refactor data path trace for telemetryv4.8.4-20240427

17 files changed, 238 insertions, 273 deletions

diff --git a/service/include/sc_app.h b/service/include/sc_app.h
index c03fa6b..b5cb9de 100644
--- a/service/include/sc_app.h
+++ b/service/include/sc_app.h
@@ -20,6 +20,7 @@ struct app
     char mntfile[MR_STRING_MAX];
     /* 监控app是否存活*/
     int fd;
+    uint8_t job_id_used;
     /* 虚设备管理模块在APP中的保存的上下文 */
     void * pme_vdev_main;
 };
diff --git a/service/include/sc_trace.h b/service/include/sc_trace.h
index 084eae2..c3d3f2d 100644
--- a/service/include/sc_trace.h
+++ b/service/include/sc_trace.h
@@ -6,6 +6,7 @@
 #include "dp_trace.h"
 #include "sc_app.h"
 #include "sc_common.h"
+#include <adapter_define.h>
 #include <mrb_define.h>
 
 #define MR_TRACE_APPSYM "mrzcpd"
@@ -18,8 +19,6 @@ void hook_rte_pktmbuf_free_bulk(struct rte_mbuf ** mbufs, unsigned int count);
 /* Generate and store the trace information for pkt parser */
 static inline void gen_store_trace_info_pkt_parser(struct rte_node * node, struct rte_mbuf * mbuf)
 {
-    struct dp_trace_record_meta meta = {.appsym=MR_TRACE_APPSYM, .module=node->name};
-
     struct mrb_metadata * mrb_meta = mrbuf_cz_data(mbuf, MR_NODE_CTRLZONE_ID);
     struct pkt_parser_result * pkt_parser_result = &mrb_meta->pkt_parser_result;
 
@@ -39,14 +38,14 @@ static inline void gen_store_trace_info_pkt_parser(struct rte_node * node, struc
     len += snprintf(str_record + len, sizeof(str_record) - len, "]");
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
 /* Generate and store the trace information for rte mbuf */
 static inline void gen_store_trace_info_rte_mbuf(struct rte_node * node, struct rte_mbuf * mbuf)
 {
-    struct dp_trace_record_meta meta = {.appsym=MR_TRACE_APPSYM, .module=node->name};
-
     /* Populate the rte mbuf information */
     char str_record[MR_STRING_MAX];
 
@@ -59,14 +58,14 @@ static inline void gen_store_trace_info_rte_mbuf(struct rte_node * node, struct
 #endif
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
 /* Generate and store the trace information for Sid list */
 static inline void gen_store_trace_info_sid_list(struct rte_node * node, struct rte_mbuf * mbuf)
 {
-    struct dp_trace_record_meta meta = {.appsym=MR_TRACE_APPSYM, .module=node->name};
-
     /* Populate the Sid list */
     struct mrb_metadata * mrb_meta = mrbuf_cz_data(mbuf, MR_NODE_CTRLZONE_ID);
     uint16_t sids[RTE_DIM(mrb_meta->sid_list.sids)];
@@ -85,6 +84,8 @@ static inline void gen_store_trace_info_sid_list(struct rte_node * node, struct
     len += snprintf(str_record + len, sizeof(str_record) - len, "]]");
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -109,25 +110,14 @@ static inline int embed_sid_info(struct rte_mbuf * mbuf, char * str_record, int
     return len;
 }
 
-
 /* Generate and store the trace information for rx */
 static inline void gen_store_trace_info_rx(struct rte_node * node, struct rte_mbuf * mbuf,
                                            struct mr_dev_desc * dev_desc, uint16_t queue_id)
 {
-    struct dp_trace_record_meta meta = {.appsym=MR_TRACE_APPSYM, .module=node->name};
-
     /* Populate the next node information */
     char str_record[MR_STRING_MAX];
-    int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[0]->name);
-
-    /* Populate the core id information */
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", core:%u", rte_lcore_id());
-
-    /* Populate the queue information */
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", qid:%u", queue_id);
-
-    /* Populate the hash information */
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", hash:%u", mbuf->hash.usr);
+    int len = snprintf(str_record, sizeof(str_record), "next node:%s, core:%u, qid:%u, hash:%u", node->nodes[0]->name,
+                       rte_lcore_id(), queue_id, mbuf->hash.usr);
 
     /* Populate the nf create infomation */
     struct mrb_metadata * mrb_meta = mrbuf_cz_data(mbuf, MR_NODE_CTRLZONE_ID);
@@ -137,6 +127,22 @@ static inline void gen_store_trace_info_rx(struct rte_node * node, struct rte_mb
     }
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
+    dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
+}
+
+/* Generate and store the telemetry information for rx */
+static inline void gen_store_telemetry_info_rx(struct rte_node * node, struct rte_mbuf * mbuf,
+                                               struct mr_dev_desc * dev_desc, uint16_t queue_id)
+{
+    /* Populate the next node information */
+    char str_record[MR_STRING_MAX];
+    snprintf(str_record, sizeof(str_record), "interface(rx) = %s, qid = %u", dev_desc->symbol, queue_id);
+
+    /* Emit the trace record */
+    struct dp_trace_record_meta meta = {.measurement_type = DP_TRACE_MEASUREMENT_TYPE_TELEMETRY,
+                                        .appsym = MR_TRACE_APPSYM};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -144,21 +150,52 @@ static inline void gen_store_trace_info_rx(struct rte_node * node, struct rte_mb
 static inline void gen_store_trace_info_tx(struct rte_node * node, struct rte_mbuf * mbuf,
                                            struct mr_dev_desc * dev_desc, uint16_t queue_id)
 {
-    struct dp_trace_record_meta meta = {.appsym=MR_TRACE_APPSYM, .module=node->name};
-
     /* Populate the port information */
     char str_record[MR_STRING_MAX];
-    int len = snprintf(str_record, sizeof(str_record), "tx:%u,%s", dev_desc->port_id, dev_desc->symbol);
+    snprintf(str_record, sizeof(str_record), "tx:%u,%s, core:%u, qid:%u, hash:%u", dev_desc->port_id, dev_desc->symbol,
+             rte_lcore_id(), queue_id, mbuf->hash.usr);
+
+    /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
+    dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
+}
 
-    /* Populate the core id information */
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", core:%u", rte_lcore_id());
+/* Generate and store the telemetry information for tx */
+static inline void gen_store_telemetry_info_tx(struct rte_node * node, struct rte_mbuf * mbuf,
+                                               struct mr_dev_desc * dev_desc, uint16_t queue_id)
+{
+    /* Populate the port information */
+    char str_record[MR_STRING_MAX];
+    snprintf(str_record, sizeof(str_record), "interface(tx) = %s, qid = %u", dev_desc->symbol, queue_id);
 
-    /* Populate the queue information */
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", qid:%u", queue_id);
+    /* Emit the trace record */
+    struct dp_trace_record_meta meta = {.measurement_type = DP_TRACE_MEASUREMENT_TYPE_TELEMETRY,
+                                        .appsym = MR_TRACE_APPSYM};
+    dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
+}
 
-    /* Populate the hash information */
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", hash:%u", mbuf->hash.usr);
+/* Generate and store the telemetry information */
+static __rte_always_inline void gen_store_telemetry_info_adapter(struct rte_mbuf * mbuf)
+{
+    char str_record[MR_STRING_MAX];
+    struct mrb_metadata * mrb_meta = mrbuf_cz_data(mbuf, MR_NODE_CTRLZONE_ID);
+    switch (mrb_meta->adapter_type)
+    {
+    case ADAPTER_TYPE_VWIRE:
+        snprintf(str_record, sizeof(str_record), "vwire_id = %u, traffic link id = %u", mrb_meta->adapter_id,
+                 mrb_meta->traffic_link_id);
+        break;
+    case ADAPTER_TYPE_EF:
+        snprintf(str_record, sizeof(str_record), "ef_id = %u, traffic link id = %u", mrb_meta->adapter_id,
+                 mrb_meta->traffic_link_id);
+        break;
+    default:
+        break;
+    }
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {.measurement_type = DP_TRACE_MEASUREMENT_TYPE_TELEMETRY,
+                                        .appsym = MR_TRACE_APPSYM};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
diff --git a/service/src/app.c b/service/src/app.c
index 764549a..e21d1a3 100644
--- a/service/src/app.c
+++ b/service/src/app.c
@@ -186,6 +186,8 @@ int __instance_alive_handler(const struct rte_mp_msg * msg, const void * peer)
     epoll_add_event(app_main->epoll_fd, app_object->fd, EPOLLIN);
 
     __send_app_register_response(msg, peer, (const char *)(reg_msg->symbol), true, NULL);
+
+    MR_INFO("Application %s registe successfully.", app_object->symbol);
     return 0;
 
 error:
diff --git a/service/src/dp_trace.c b/service/src/dp_trace.c
index e229b68..35949d0 100644
--- a/service/src/dp_trace.c
+++ b/service/src/dp_trace.c
@@ -8,79 +8,17 @@
 
 struct dp_trace_config
 {
-    bool enable;
-    char file_path[PATH_MAX];
-    unsigned int file_max_size; // KB
-    unsigned int trace_merge_timeout;
-
-    struct dp_trace_job_desc desc[DP_TRACE_JOB_NUM_MAX];
+    uint16_t enable;
 };
 
-int __data_trace_path_handler(const struct rte_mp_msg * msg, const void * peer);
+int __dp_trace_path_handler(const struct rte_mp_msg * msg, const void * peer);
+static void dp_trace_telemetry_unregister(struct app_main * app_main, struct app * app, void * arg);
 
 void load_dp_trace_config(struct sc_main * sc, struct dp_trace_config * dp_trace_config)
 {
-    int ret = 0;
-
-    unsigned int enable = 0;
+    unsigned int enable = 1;
     MESA_load_profile_uint_nodef(sc->local_dyfile, "dp_trace_rule", "enable", &enable);
-    dp_trace_config->enable = (enable == 0) ? false : true;
-
-    char dp_trace_file_path[MR_SYMBOL_MAX] = "/tmp/dp_trace.pcapng";
-    MESA_load_profile_string_nodef(sc->local_dyfile, "dp_trace_rule", "dp_trace_file_path", dp_trace_file_path,
-                                   sizeof(dp_trace_file_path));
-    snprintf(dp_trace_config->file_path, sizeof(dp_trace_config->file_path), "%s", dp_trace_file_path);
-
-    unsigned int dp_trace_file_max_size_in_KB = 256 * 1024;
-    MESA_load_profile_uint_nodef(sc->local_dyfile, "dp_trace_rule", "dp_trace_file_max_size_in_KB",
-                                 &dp_trace_file_max_size_in_KB);
-    dp_trace_config->file_max_size = dp_trace_file_max_size_in_KB;
-
-    unsigned int dp_trace_merge_timeout = 30;
-    MESA_load_profile_uint_nodef(sc->local_dyfile, "dp_trace_rule", "dp_trace_merge_timeout", &dp_trace_merge_timeout);
-    dp_trace_config->trace_merge_timeout = dp_trace_merge_timeout;
-
-    for (int rule_index = 0; rule_index < DP_TRACE_JOB_NUM_MAX; rule_index++)
-    {
-        unsigned int enable = 1;
-        char bpf_expr[MR_BPF_EXPRESSION_MAX] = {};
-        unsigned int pkt_cnt_max = 0;
-        unsigned int sampling = 1;
-        unsigned int snaplen = UINT32_MAX;
-
-        char dp_trace_section[MR_SYMBOL_MAX] = {};
-        snprintf(dp_trace_section, sizeof(dp_trace_section), "dp_trace_rule:%d", rule_index);
-
-        MESA_load_profile_uint_nodef(sc->local_dyfile, dp_trace_section, "enable", &enable);
-
-        ret =
-            MESA_load_profile_string_nodef(sc->local_dyfile, dp_trace_section, "bpf_expr", bpf_expr, sizeof(bpf_expr));
-        if (ret < 0)
-        {
-            continue;
-        }
-
-        MESA_load_profile_uint_nodef(sc->local_dyfile, dp_trace_section, "pkt_cnt_max", &pkt_cnt_max);
-        MESA_load_profile_uint_nodef(sc->local_dyfile, dp_trace_section, "sampling", &sampling);
-        MESA_load_profile_uint_nodef(sc->local_dyfile, dp_trace_section, "snaplen", &snaplen);
-
-        struct dp_trace_job_desc * desc = (struct dp_trace_job_desc *)&dp_trace_config->desc;
-        struct dp_trace_job_desc * desc_i = &desc[rule_index];
-        desc_i->enable = (enable == 0) ? false : true;
-        snprintf(desc_i->bpf_expr, sizeof(desc_i->bpf_expr), "%s", bpf_expr);
-
-        // The maximum number of packets is limited to each core
-        // assert(sc->nr_io_thread != 0);
-        // pkt_cnt_max = pkt_cnt_max / sc->nr_io_thread;
-        // if (pkt_cnt_max % sc->nr_io_thread != 0)
-        // {
-        //     pkt_cnt_max++;
-        // }
-
-        desc_i->pkt_cnt_max = pkt_cnt_max;
-        desc_i->sampling = (sampling == 0) ? 1 : sampling;
-        desc_i->snaplen = (snaplen == 0) ? UINT32_MAX : snaplen;
-    }
+    dp_trace_config->enable = enable;
 }
 
 void dp_trace_config_apply(struct sc_main * sc)
@@ -89,98 +27,15 @@ void dp_trace_config_apply(struct sc_main * sc)
         return;
 
     struct dp_trace_config dp_trace_config = {};
-
-    struct dp_trace_job_desc * desc = (struct dp_trace_job_desc *)&dp_trace_config.desc;
-
     MR_INFO("Loading data path trace configuration file...");
-    load_dp_trace_config(sc, &dp_trace_config);
-
-    if (sc->trace->inst->enable == false && dp_trace_config.enable == false)
-    {
-        MR_INFO("continue to disable data path trace");
-        goto end;
-    }
-    else if (sc->trace->inst->enable == true && dp_trace_config.enable == false)
-    {
-        MR_INFO("disable data path trace");
-        dp_trace_stop(sc->trace);
-        dp_trace_jobs_destroy(sc->trace, DP_TRACE_ALL_JOBS);
-        goto end;
-    }
 
-    if (sc->trace->inst->enable == false && dp_trace_config.enable == true)
+    load_dp_trace_config(sc, &dp_trace_config);
+    if (sc->trace->inst->enable != dp_trace_config.enable)
     {
-        struct dp_trace_instance * inst = sc->trace->inst;
-        snprintf(inst->trace_file_path, sizeof(inst->trace_file_path), "%s", dp_trace_config.file_path);
-        inst->trace_file_max_size = dp_trace_config.file_max_size;
-        inst->trace_merge_timeout = dp_trace_config.trace_merge_timeout;
-
-        if (!dp_trace_start(sc->trace))
-        {
-            MR_INFO("failed to enable data path trace");
-            goto end;
-        }
-        MR_INFO("enable data path trace");
+        MR_INFO("Modify trace status from %u to %u", sc->trace->inst->enable, dp_trace_config.enable);
+        sc->trace->inst->enable = dp_trace_config.enable;
     }
 
-    struct dp_trace_job_ctx * job_ctx = sc->trace->inst->job_ctx;
-    for (unsigned int i = 0; i < DP_TRACE_JOB_NUM_MAX; i++)
-    {
-        // The job that was not in the system originally is not in the configuration now.
-        if (job_ctx[i].used == false && (desc[i].enable == false || strlen(desc[i].bpf_expr) == 0))
-        {
-            continue;
-        }
-
-        // The job that was not originally in the system is now included in the configuration.
-        if (job_ctx[i].used == false && desc[i].enable && strlen(desc[i].bpf_expr) != 0)
-        {
-            MR_INFO(" [add or enable rule %u] bpf_expr: %s;", i, desc[i].bpf_expr);
-            dp_trace_job_add(sc->trace, &desc[i], i);
-            continue;
-        }
-
-        // The job  that was originally in the system is also included in the configuration now,
-        // but the content is different.
-        if (job_ctx[i].used == true && desc[i].enable && strlen(desc[i].bpf_expr) != 0)
-        {
-            struct dp_trace_job_desc * job_ctx_i_desc = &job_ctx[i].desc;
-            if (strcmp(job_ctx_i_desc->bpf_expr, desc[i].bpf_expr) != 0)
-            {
-                MR_INFO(" [mod rule %u] bpf_expr: %s;", i, desc[i].bpf_expr);
-                dp_trace_jobs_destroy(sc->trace, job_ctx[i].job_id);
-                dp_trace_job_add(sc->trace, &desc[i], i);
-            }
-            else
-            {
-                if (job_ctx_i_desc->pkt_cnt_max != desc[i].pkt_cnt_max)
-                {
-                    job_ctx_i_desc->pkt_cnt_max = desc[i].pkt_cnt_max;
-                }
-
-                if (job_ctx_i_desc->sampling != desc[i].sampling)
-                {
-                    job_ctx_i_desc->sampling = desc[i].sampling;
-                }
-
-                if (job_ctx_i_desc->snaplen != desc[i].snaplen)
-                {
-                    job_ctx_i_desc->snaplen = desc[i].snaplen;
-                }
-            }
-            continue;
-        }
-
-        // The job that originally existed in the system is not in the configuration now.
-        if (job_ctx[i].used == true && (desc[i].enable == false || strlen(desc[i].bpf_expr) == 0))
-        {
-            MR_INFO(" [del or disable rule %u] bpf_expr: %s;", i, job_ctx[i].desc.bpf_expr);
-            dp_trace_jobs_destroy(sc->trace, job_ctx[i].job_id);
-            continue;
-        }
-    }
-
-end:
     MR_INFO("Loading data path trace configuration file is completed.");
 }
 
@@ -199,21 +54,73 @@ int mr_dp_trace_init(struct sc_main * sc)
 
     dp_trace_config_apply(sc);
 
-    rte_mp_action_register("data_path_trace", __data_trace_path_handler);
+    rte_mp_action_register(DP_TRACE_MP_MSG_NAME, __dp_trace_path_handler);
+    app_event_handler_register(sc->app_main, APP_EV_TYPE_UNREGISTER, dp_trace_telemetry_unregister, NULL);
 
     return 0;
 }
 
-int __data_trace_path_handler(const struct rte_mp_msg * msg, const void * peer)
+int __dp_trace_path_handler(const struct rte_mp_msg * msg, const void * peer)
 {
-    struct rte_mp_msg resp;
-    snprintf(resp.name, sizeof(resp.name), "%s", "data_path_trace");
+    struct dp_trace_instance * inst = sc_main_get()->trace->inst;
+    struct dp_trace_job_ctx * job_ctx = inst->job_ctx;
+    int ret = 0;
+
+    struct rte_mp_msg resp = {};
+    snprintf(resp.name, sizeof(resp.name), "%s", DP_TRACE_MP_MSG_NAME);
+
+    struct dp_trace_resp * dp_trace_resp = (struct dp_trace_resp *)resp.param;
+    resp.len_param = sizeof(struct dp_trace_resp);
 
-    uintptr_t ptr_address = (uintptr_t)sc_main_get()->trace->inst;
-    resp.len_param = sizeof(ptr_address);
-    memcpy(resp.param, &ptr_address, sizeof(ptr_address));
+    struct dp_trace_req * dp_trace_req = (struct dp_trace_req *)msg->param;
+    if (dp_trace_req->action == DP_TRACE_INSTANCE_GET)
+    {
+        dp_trace_resp->errcode = DP_TRACE_SUCCESS;
+        dp_trace_resp->trace_instance = inst;
+        goto send_resp;
+    }
+
+    if (dp_trace_req->action == DP_TRACE_JOB_ID_USED_GET)
+    {
+        job_bitmap_t jobs_id_uesd = 0;
+        for (unsigned int i = 0; i < DP_TRACE_JOB_NUM_MAX; i++)
+        {
+            if (job_ctx[i].used == true)
+            {
+                jobs_id_uesd |= job_ctx[i].job_id;
+            }
+        }
+
+        dp_trace_resp->jobs_id_used_get = jobs_id_uesd;
+        dp_trace_resp->errcode = DP_TRACE_SUCCESS;
+        goto send_resp;
+    }
 
-    int ret = rte_mp_reply(&resp, peer);
+    if (dp_trace_req->action == DP_TRACE_JOB_ADD)
+    {
+        dp_trace_resp->errcode = dp_trace_job_add(sc_main_get()->trace, dp_trace_req->desc);
+        if (dp_trace_resp->errcode == DP_TRACE_SUCCESS)
+        {
+            unsigned int rule_index = dp_trace_req->desc->rule_index;
+            dp_trace_resp->job_id_add = job_ctx[rule_index].job_id;
+
+            struct app * app_object = app_lookup_by_symbol(sc_main_get(), (const char *)dp_trace_req->appsym);
+            assert(app_object != NULL);
+            app_object->job_id_used = 1;
+        }
+        goto send_resp;
+    }
+
+    if (dp_trace_req->action == DP_TRACE_JOB_DESTROY)
+    {
+        job_bitmap_t jobs_id = dp_trace_req->jobs_id_destroy;
+        jobs_id = jobs_id & 0xffff;
+        dp_trace_jobs_destroy(sc_main_get()->trace, jobs_id);
+        dp_trace_resp->errcode = DP_TRACE_SUCCESS;
+    }
+
+send_resp:
+    ret = rte_mp_reply(&resp, peer);
     if (ret < 0)
     {
         MR_WARNING("Failed to execute rte_mp_reply:%s", rte_strerror(rte_errno));
@@ -222,17 +129,32 @@ int __data_trace_path_handler(const struct rte_mp_msg * msg, const void * peer)
     return 0;
 }
 
+static void dp_trace_telemetry_unregister(struct app_main * app_main, struct app * app, void * arg)
+{
+    if (app->job_id_used == 0)
+    {
+        return;
+    }
+
+    struct dp_trace_process * dp_trace_process = sc_main_get()->trace;
+    dp_trace_jobs_destroy(dp_trace_process, DP_TRACE_ALL_JOBS);
+}
+
 cJSON * dp_trace_monit_loop(struct sc_main * sc)
 {
     cJSON * json_root = cJSON_CreateObject();
     struct dp_trace_process * trace = sc->trace;
 
     struct dp_trace_stat monit_statistics = {};
+    uint64_t filter_exec_hit = 0;
 
     for (unsigned int i = 0; i < RTE_MAX_LCORE; i++)
     {
         struct dp_trace_stat * statistics_i = &trace->statistics[i];
-        monit_statistics.filter_exec_hit += statistics_i->filter_exec_hit;
+        for (unsigned int j = 0; j < DP_TRACE_JOB_NUM_MAX; j++)
+        {
+            filter_exec_hit += statistics_i->filter_exec_hit[j];
+        }
         monit_statistics.filter_exec_miss += statistics_i->filter_exec_miss;
         monit_statistics.reach_pkt_cnt_limit += statistics_i->reach_pkt_cnt_limit;
         monit_statistics.record_buf_alloc_failed_no_mem += statistics_i->record_buf_alloc_failed_no_mem;
@@ -244,11 +166,9 @@ cJSON * dp_trace_monit_loop(struct sc_main * sc)
         monit_statistics.record_emit_success += statistics_i->record_emit_success;
     }
 
-    struct dp_trace_saving_stat * saving_statistics = &trace->saving_statistics;
-
     cJSON_AddBoolToObject(json_root, "trace_enable", trace->inst->enable);
 
-    cJSON_AddNumberToObject(json_root, "filter_exec_hit", monit_statistics.filter_exec_hit);
+    cJSON_AddNumberToObject(json_root, "filter_exec_hit", filter_exec_hit);
     cJSON_AddNumberToObject(json_root, "filter_exec_miss", monit_statistics.filter_exec_miss);
 
     cJSON_AddNumberToObject(json_root, "reach_pkt_cnt_limit", monit_statistics.reach_pkt_cnt_limit);
@@ -266,13 +186,6 @@ cJSON * dp_trace_monit_loop(struct sc_main * sc)
                             monit_statistics.record_emit_failed_no_space_in_buf);
     cJSON_AddNumberToObject(json_root, "record_emit_success", monit_statistics.record_emit_success);
 
-    cJSON_AddNumberToObject(json_root, "save_to_file_failed_other", saving_statistics->save_to_file_failed_other);
-    cJSON_AddNumberToObject(json_root, "save_to_file_failed_at_pcapng_format",
-                            saving_statistics->save_to_file_failed_at_pcapng_format);
-    cJSON_AddNumberToObject(json_root, "save_to_file_failed_at_write_to_disk",
-                            saving_statistics->save_to_file_failed_at_write_to_disk);
-    cJSON_AddNumberToObject(json_root, "save_to_file_success", saving_statistics->save_to_file_success);
-
     return json_root;
 }
 
@@ -282,7 +195,6 @@ void hook_rte_pktmbuf_free(struct rte_mbuf * m)
         return;
     struct dp_trace_process * trace = sc_main_get()->trace;
     dp_trace_record_write(trace, m, rte_lcore_id());
-    dp_trace_record_flush(trace);
     rte_pktmbuf_free(m);
 }
 
@@ -295,6 +207,5 @@ void hook_rte_pktmbuf_free_bulk(struct rte_mbuf ** mbufs, unsigned int count)
             continue;
         dp_trace_record_write(trace, mbufs[idx], rte_lcore_id());
     }
-    dp_trace_record_flush(trace);
     rte_pktmbuf_free_bulk(mbufs, count);
 }
\ No newline at end of file
diff --git a/service/src/node_bfd.c b/service/src/node_bfd.c
index d039ad2..4d35f33 100644
--- a/service/src/node_bfd.c
+++ b/service/src/node_bfd.c
@@ -152,8 +152,6 @@ static __rte_always_inline void gen_store_trace_info(struct rte_node * node, str
                                                      uint16_t next_node_index, int expect_result,
                                                      uint16_t bfd_session_id, uint8_t bfd_state)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
     int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[next_node_index]->name);
@@ -175,6 +173,8 @@ static __rte_always_inline void gen_store_trace_info(struct rte_node * node, str
     len += snprintf(str_record + len, sizeof(str_record) - len, ", bfd state:%u", bfd_state);
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -251,7 +251,7 @@ static __rte_always_inline uint16_t bfd_node_process(struct rte_graph * graph, s
 
     node_enqueue:
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info(node, mbuf, next_node_index, expect_result, bfd_session_id, bfd_state);
         }
diff --git a/service/src/node_bridge.c b/service/src/node_bridge.c
index 2790ae6..cf018a3 100644
--- a/service/src/node_bridge.c
+++ b/service/src/node_bridge.c
@@ -227,7 +227,6 @@ static __rte_always_inline void gen_store_trace_info_original(struct rte_node *
 {
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
-
     assert(match_result < BRIDGE_MATCH_MAX);
     int len = snprintf(str_record, sizeof(str_record), ", rsn:%s", bridge_match_result_str[match_result]);
 
@@ -239,7 +238,8 @@ static __rte_always_inline void gen_store_trace_info_original(struct rte_node *
     }
 
     /* Emit the trace record */
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -247,7 +247,6 @@ static __rte_always_inline void gen_store_trace_info_original(struct rte_node *
 static __rte_always_inline void gen_store_trace_info_clone(struct rte_node * node, struct rte_mbuf * mbuf,
                                                            uint16_t next_node_index)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
 
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
@@ -256,6 +255,8 @@ static __rte_always_inline void gen_store_trace_info_clone(struct rte_node * nod
              node->nodes[next_node_index]->name, mrb_meta->port_egress);
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -365,7 +366,7 @@ static __rte_always_inline uint16_t bridge_node_process(struct rte_graph * graph
             dp_trace_filter_exec(sc_main_get()->trace, mbuf_clone, 0, rte_lcore_id());
 
             /* Check if tracing is enabled for the current Mbuf */
-            if (unlikely(dp_trace_record_can_emit(mbuf)))
+            if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
             {
                 gen_store_trace_info_clone(node, mbuf_clone, BRIDGE_NEXT_ETH_EGRESS);
             }
@@ -376,7 +377,7 @@ static __rte_always_inline uint16_t bridge_node_process(struct rte_graph * graph
 
     node_enqueue:
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_original(node, mbuf, next_node_index, &stats, match_result);
         }
diff --git a/service/src/node_classifier.c b/service/src/node_classifier.c
index 88b65ee..d4e87e3 100644
--- a/service/src/node_classifier.c
+++ b/service/src/node_classifier.c
@@ -364,8 +364,6 @@ static __rte_always_inline void gen_store_trace_info(struct rte_node * node, str
                                                      struct match_result_engine * result, uint8_t ignore_icmp,
                                                      enum cls_excpt_reason excpt_reason)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
     int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[next_node_index]->name);
@@ -401,6 +399,8 @@ static __rte_always_inline void gen_store_trace_info(struct rte_node * node, str
     }
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -515,7 +515,7 @@ static __rte_always_inline uint16_t classifier_node_process(struct rte_graph * g
             }
 
             /* Check if tracing is enabled for the current Mbuf */
-            if (unlikely(dp_trace_record_can_emit(pending_mbufs[pkt_idx])))
+            if (unlikely(dp_trace_record_can_emit(pending_mbufs[pkt_idx], DP_TRACE_MEASUREMENT_TYPE_TRACE)))
             {
                 gen_store_trace_info(node, pending_mbufs[pkt_idx], CLASSIFIER_NEXT_FORWARDER, &stats, &results[pkt_idx],
                                      ignore_icmp_pkts[pkt_idx], excpt_reason);
@@ -600,7 +600,7 @@ static __rte_always_inline uint16_t classifier_node_process(struct rte_graph * g
         }
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf0)))
+        if (unlikely(dp_trace_record_can_emit(mbuf0, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info(node, mbuf0, CLASSIFIER_NEXT_FORWARDER, &stats, &result, ignore_icmp_pkt,
                                  excpt_reason);
diff --git a/service/src/node_eth_egress.c b/service/src/node_eth_egress.c
index 7614cff..dd77595 100644
--- a/service/src/node_eth_egress.c
+++ b/service/src/node_eth_egress.c
@@ -81,17 +81,14 @@ int node_eth_egress_init(struct node_manager_main * node_mgr_main)
 static __rte_always_inline void gen_store_trace_info_egress(struct rte_node * node, struct rte_mbuf * mbuf,
                                                             uint16_t next_node_index)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
-    int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[next_node_index]->name);
-
-    /* Populate the egress port information */
     struct mrb_metadata * mrb_meta = mrbuf_cz_data(mbuf, MR_NODE_CTRLZONE_ID);
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", tx:%u", mrb_meta->port_egress);
+    snprintf(str_record, sizeof(str_record), "next node:%s, tx:%u", node->nodes[next_node_index]->name,
+             mrb_meta->port_egress);
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -124,7 +121,7 @@ static __rte_always_inline uint16_t eth_egress_node_process(struct rte_graph * g
 
 #if 0
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_egress(node, mbuf, next_node_index);
         }
diff --git a/service/src/node_eth_ingress.c b/service/src/node_eth_ingress.c
index ad7468a..bf3f444 100644
--- a/service/src/node_eth_ingress.c
+++ b/service/src/node_eth_ingress.c
@@ -409,14 +409,11 @@ static __rte_always_inline void gen_store_trace_info_ingress(struct rte_node * n
                                                              struct mr_dev_desc * dev_desc,
                                                              enum eth_ingress_drop_reason drop_reason)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
     int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[next_node_index]->name);
 
     /* Populate the reason for next node */
-
     switch (next_node_index)
     {
     case ETH_INGRESS_NEXT_PKT_DROP: {
@@ -456,6 +453,8 @@ static __rte_always_inline void gen_store_trace_info_ingress(struct rte_node * n
     }
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -536,7 +535,7 @@ static __rte_always_inline uint16_t eth_ingress_node_process(struct rte_graph *
     node_enqueue:
 #if 1
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_ingress(node, mbuf, next_node_index, &stats, dev_desc, drop_reason);
         }
diff --git a/service/src/node_etherfabric.c b/service/src/node_etherfabric.c
index da856b7..1174e6a 100644
--- a/service/src/node_etherfabric.c
+++ b/service/src/node_etherfabric.c
@@ -498,8 +498,6 @@ static __rte_always_inline void gen_store_trace_info_ingress(struct rte_node * n
                                                              uint16_t next_node_index, uint16_t prepend_sid,
                                                              enum ef_ingress_drop_reason drop_reason)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
     struct mrb_metadata * mrb_meta = mrbuf_cz_data(mbuf, MR_NODE_CTRLZONE_ID);
@@ -522,6 +520,8 @@ static __rte_always_inline void gen_store_trace_info_ingress(struct rte_node * n
     }
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -619,13 +619,18 @@ static __rte_always_inline uint16_t ef_ingress_node_process(struct rte_graph * g
 
     node_enqueue:
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_ingress(node, mbuf, next_node_index, prepend_sid, drop_reason);
             // gen_store_trace_info_sid_list(node, mbuf);
             //  gen_store_trace_info_rte_mbuf(node, mbuf);
         }
 
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TELEMETRY)))
+        {
+            gen_store_telemetry_info_adapter(mbuf);
+        }
+
         /* Judge the next index whether to change */
         if (unlikely(batch_next_node_index != next_node_index))
         {
@@ -750,8 +755,6 @@ static __rte_always_inline void gen_store_trace_info_egress(struct rte_node * no
                                                             uint16_t next_node_index,
                                                             enum ef_egress_drop_reason drop_reason)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
     int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[next_node_index]->name);
@@ -778,6 +781,8 @@ static __rte_always_inline void gen_store_trace_info_egress(struct rte_node * no
     }
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -861,7 +866,7 @@ static __rte_always_inline uint16_t ef_egress_node_process(struct rte_graph * gr
         }
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_egress(node, mbuf, next_node_index, drop_reason);
             // gen_store_trace_info_rte_mbuf(node, mbuf);
diff --git a/service/src/node_forwarder.c b/service/src/node_forwarder.c
index 41bcf5a..ba5aee3 100644
--- a/service/src/node_forwarder.c
+++ b/service/src/node_forwarder.c
@@ -106,8 +106,6 @@ static __rte_always_inline void gen_store_trace_info(struct rte_node * node, str
                                                      uint16_t next_node_index, struct forwarder_stats * stats,
                                                      enum forwarder_drop_reason drop_reason)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
     int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[next_node_index]->name);
@@ -130,6 +128,8 @@ static __rte_always_inline void gen_store_trace_info(struct rte_node * node, str
     len += embed_sid_info(mbuf, str_record + len, sizeof(str_record) - len);
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -206,7 +206,7 @@ static __rte_always_inline uint16_t forwarder_node_process(struct rte_graph * gr
 
     node_enqueue:
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info(node, mbuf, next_node_index, &stats, drop_reason);
             // gen_store_trace_info_sid_list(node, mbuf);
diff --git a/service/src/node_health_check.c b/service/src/node_health_check.c
index 3a5662c..f8f9afd 100644
--- a/service/src/node_health_check.c
+++ b/service/src/node_health_check.c
@@ -649,21 +649,14 @@ static __rte_always_inline void gen_store_trace_info_ask_node(struct rte_node *
                                                               uint16_t next_node_index,
                                                               struct health_check_session * session)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
-    int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[next_node_index]->name);
-
-    /* Populate the core id */
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", core:%u", rte_lcore_id());
-
-    /* Populate the health check session infomation */
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", session name:%s", session->name);
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", listening dev:%u,%s", session->port_id,
-                    session->device);
+    snprintf(str_record, sizeof(str_record), "next node:%s, core:%u, session name:%s, listening dev:%u,%s",
+             node->nodes[next_node_index]->name, rte_lcore_id(), session->name, session->port_id, session->device);
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -708,7 +701,7 @@ static __rte_always_inline uint16_t health_check_ask_node_process(struct rte_gra
         dp_trace_filter_exec(sc_main_get()->trace, mbuf, 0, rte_lcore_id());
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_ask_node(node, mbuf, HEALTH_CHECK_ASK_NEXT_ETH_EGRESS, session);
         }
@@ -726,7 +719,7 @@ static __rte_always_inline uint16_t health_check_ask_node_process(struct rte_gra
     for (int i = 0; i < nr_mbufs; i++)
     {
         struct rte_mbuf * mbuf = (struct rte_mbuf *)node->objs[i];
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
             gen_store_trace_info_rte_mbuf(node, mbuf);
     }
 
@@ -786,7 +779,8 @@ static __rte_always_inline void gen_store_trace_info_answer_node(struct rte_node
     }
 
     /* Emit the trace record */
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -834,7 +828,7 @@ static __rte_always_inline uint16_t health_check_deal_answer_node_process(struct
         }
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_answer_node(node, mbuf, HEALTH_CHECK_DEAL_ANSWER_NEXT_PKT_DROP, &stats, session,
                                              excpt_reason);
diff --git a/service/src/node_lb.c b/service/src/node_lb.c
index 552200f..f86fcbe 100644
--- a/service/src/node_lb.c
+++ b/service/src/node_lb.c
@@ -454,7 +454,6 @@ int lb_init(struct sc_main * sc)
 static __rte_always_inline void gen_store_trace_info(struct rte_node * node, struct rte_mbuf * mbuf,
                                                      uint16_t next_node_index, struct lb_group * lb_group)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
 
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
@@ -477,6 +476,8 @@ static __rte_always_inline void gen_store_trace_info(struct rte_node * node, str
         snprintf(str_record + len, sizeof(str_record) - len, ", cur sid:%u, lb id:%u", lb_group->sid, lb_group->index);
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -537,7 +538,7 @@ static __rte_always_inline uint16_t lb_node_process(struct rte_graph * graph, st
         }
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info(node, mbuf, next_node_index, lb_group);
         }
diff --git a/service/src/node_phydev.c b/service/src/node_phydev.c
index ba5b219..39c16f0 100644
--- a/service/src/node_phydev.c
+++ b/service/src/node_phydev.c
@@ -5,6 +5,7 @@
 #include <rte_graph_worker.h>
 #include <rte_mbuf.h>
 
+#include <cJSON.h>
 #include <metadata_define.h>
 #include <rte_memcpy.h>
 #include <sc_node.h>
@@ -423,7 +424,7 @@ static __rte_always_inline uint16_t dpdk_msgpack_dev_rx_node_process(struct rte_
         dp_trace_filter_exec(sc_main_get()->trace, mbuf, 0);
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_rx(node, mbuf, ctx->dev_desc, graph->id);
         }
@@ -530,11 +531,16 @@ static __rte_always_inline uint16_t dpdk_dev_rx_node_process(struct rte_graph *
         dp_trace_filter_exec(sc_main_get()->trace, mbuf, 0, rte_lcore_id());
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_rx(node, mbuf, dev_desc, qid);
             gen_store_trace_info_pkt_parser(node, mbuf);
         }
+
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TELEMETRY)))
+        {
+            gen_store_telemetry_info_rx(node, mbuf, dev_desc, qid);
+        }
     }
 
     /* for test */
@@ -544,7 +550,7 @@ static __rte_always_inline uint16_t dpdk_dev_rx_node_process(struct rte_graph *
     for (unsigned int i = 0; i < node->idx; i++)
     {
         struct rte_mbuf * mbuf = (struct rte_mbuf *)node->objs[i];
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_rte_mbuf(node, mbuf);
         }
@@ -597,7 +603,7 @@ static __rte_always_inline uint16_t dpdk_msgpack_dev_tx_node_process(struct rte_
         struct rte_mbuf * mbuf = (struct rte_mbuf *)node->objs[i];
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_tx(node, mbuf, dev_desc, graph->id);
         }
@@ -647,15 +653,18 @@ static __rte_always_inline uint16_t dpdk_dev_tx_node_process(struct rte_graph *
         struct rte_mbuf * mbuf = (struct rte_mbuf *)objs[i];
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_tx(node, mbuf, dev_desc, graph->id);
             // gen_store_trace_info_rte_mbuf(node, mbuf);
-            dp_trace_record_write(sc_main_get()->trace, mbuf, rte_lcore_id());
         }
-    }
 
-    dp_trace_record_flush(sc_main_get()->trace);
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TELEMETRY)))
+        {
+            gen_store_telemetry_info_tx(node, mbuf, dev_desc, graph->id);
+        }
+        dp_trace_record_write(sc_main_get()->trace, mbuf, rte_lcore_id());
+    }
 
     /* do CF calculate */
 #if MR_PHYDEV_ENABLE_DF_CALCULATE
diff --git a/service/src/node_shmdev.c b/service/src/node_shmdev.c
index 1f02d68..d840866 100644
--- a/service/src/node_shmdev.c
+++ b/service/src/node_shmdev.c
@@ -46,8 +46,6 @@ enum packet_direction
 static __rte_always_inline void gen_store_trace_info(struct rte_node * node, struct rte_mbuf * mbuf,
                                                      struct vdev * shm_dev_desc, enum packet_direction direction)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     char str_record[MR_STRING_MAX];
     int len = snprintf(str_record, sizeof(str_record), "port:%u,%s", shm_dev_desc->port_id, shm_dev_desc->symbol);
 
@@ -57,6 +55,7 @@ static __rte_always_inline void gen_store_trace_info(struct rte_node * node, str
     }
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -126,7 +125,7 @@ uint16_t shmdev_rx_node_process(struct rte_graph * graph, struct rte_node * node
         mrb_meta->port_ingress = dev_desc->port_id;
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf0)))
+        if (unlikely(dp_trace_record_can_emit(mbuf0, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_rx(node, mbuf0, dev_desc, qid);
 
@@ -163,7 +162,7 @@ uint16_t shmdev_tx_node_process(struct rte_graph * graph, struct rte_node * node
     for (unsigned int i = 0; i < nr_pkts; i++)
     {
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbufs[i])))
+        if (unlikely(dp_trace_record_can_emit(mbufs[i], DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_tx(node, mbufs[i], dev_desc, graph->id);
             // gen_store_trace_info_rte_mbuf(node, mbufs[i]);
diff --git a/service/src/node_tera.c b/service/src/node_tera.c
index 3453631..a0e4a77 100644
--- a/service/src/node_tera.c
+++ b/service/src/node_tera.c
@@ -351,7 +351,8 @@ static __rte_always_inline void gen_store_trace_info_ingress(struct rte_node * n
     }
 
     /* Emit the trace record */
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -459,7 +460,7 @@ static __rte_always_inline uint16_t tera_ingress_node_process(struct rte_graph *
 
     node_enqueue:
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_ingress(node, mbuf, next_node_index, &stats, vlan_id, prev_vlan_couple,
                                          prev_adapter->mac_flipping, prepend_sid, drop_reason);
@@ -519,16 +520,16 @@ RTE_NODE_REGISTER(tera_ingress_node_base);
 static __rte_always_inline void gen_store_trace_info_egress(struct rte_node * node, struct rte_mbuf * mbuf,
                                                             uint16_t next_node_index)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
 
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
     struct mrb_metadata * mrb_meta = mrbuf_cz_data(mbuf, MR_NODE_CTRLZONE_ID);
-
     snprintf(str_record, sizeof(str_record), "next node:%s, tera id:%u, tx:%u", node->nodes[next_node_index]->name,
              mrb_meta->adapter_id, mrb_meta->port_egress);
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -579,7 +580,7 @@ static __rte_always_inline uint16_t tera_egress_node_process(struct rte_graph *
         mrb_meta->port_egress = prev_port_id;
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf)))
+        if (unlikely(dp_trace_record_can_emit(mbuf, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_egress(node, mbuf, TERA_EGRESS_NEXT_ETH_EGRESS);
         }
diff --git a/service/src/node_vwire.c b/service/src/node_vwire.c
index 5a64e9d..01fb74d 100644
--- a/service/src/node_vwire.c
+++ b/service/src/node_vwire.c
@@ -1,5 +1,6 @@
 #include "sc_trace.h"
 #include <cJSON.h>
+#include <rte_branch_prediction.h>
 #include <rte_debug.h>
 #include <rte_ethdev.h>
 #include <rte_ether.h>
@@ -131,7 +132,8 @@ static __rte_always_inline void gen_store_trace_info_ingress(struct rte_node * n
     }
 
     /* Emit the trace record */
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -199,12 +201,17 @@ static __rte_always_inline uint16_t vwire_ingress_node_process(struct rte_graph
         }
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf0)))
+        if (unlikely(dp_trace_record_can_emit(mbuf0, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_ingress(node, mbuf0, next0, prepend_sid);
             // gen_store_trace_info_sid_list(node, mbuf0);
         }
 
+        if (unlikely(dp_trace_record_can_emit(mbuf0, DP_TRACE_MEASUREMENT_TYPE_TELEMETRY)))
+        {
+            gen_store_telemetry_info_adapter(mbuf0);
+        }
+
         if (unlikely(next_index ^ next0))
         {
             /* Copy things successfully speculated till now */
@@ -265,7 +272,8 @@ static __rte_always_inline void gen_store_trace_info_egress(struct rte_node * no
     }
 
     /* Emit the trace record */
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
+    struct dp_trace_record_meta meta = {
+        .measurement_type = DP_TRACE_MEASUREMENT_TYPE_TRACE, .appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -329,7 +337,7 @@ static __rte_always_inline uint16_t vwire_egress_node_process(struct rte_graph *
         }
 
         /* Check if tracing is enabled for the current Mbuf */
-        if (unlikely(dp_trace_record_can_emit(mbuf0)))
+        if (unlikely(dp_trace_record_can_emit(mbuf0, DP_TRACE_MEASUREMENT_TYPE_TRACE)))
         {
             gen_store_trace_info_egress(node, mbuf0, next0);
         }