path: root/infra/packet_manager/packet_manager.c

#include <assert.h>

#include "utils_internal.h"
#include "packet_internal.h"
#include "packet_builder.h"
#include "packet_manager.h"
#include "fieldstat/fieldstat_easy.h"

#define PACKET_MANAGER_LOG_ERROR(format, ...) STELLAR_LOG_ERROR(__thread_local_logger, "packet manager", format, ##__VA_ARGS__)
#define PACKET_MANAGER_LOG_FATAL(format, ...) STELLAR_LOG_FATAL(__thread_local_logger, "packet manager", format, ##__VA_ARGS__)
#define PACKET_MANAGER_LOG_INFO(format, ...) STELLAR_LOG_INFO(__thread_local_logger, "packet manager", format, ##__VA_ARGS__)

struct node
{
    char name[64];
    uint64_t interested_tag_key_bits;
    uint64_t interested_tag_val_bits;
    on_packet_callback *node_entry;
    struct module *mod;
};

#define MAX_NODE_PER_STAGE 128
struct node_array
{
    struct node array[MAX_NODE_PER_STAGE];
    uint16_t used;
};

struct packet_manager_rte
{
    enum packet_stage curr_stage;
    struct packet_queue queue[PACKET_QUEUE_MAX];

    void *claim_arg;
    on_packet_callback *claim_cb;

    struct packet_manager_stat stat;
};

struct packet_manager
{
    uint16_t thread_num;
    struct exdata_schema *ex_sche;
    struct node_array nodes[PACKET_STAGE_MAX];
    struct packet_manager_rte *rte[MAX_THREAD_NUM];

    struct fieldstat_easy *fs;
    int fs_idx[PKT_MGR_STAT_MAX];
};

/******************************************************************************
 * utils
 ******************************************************************************/

uint64_t packet_manager_stat_get(struct packet_manager_stat *stat, enum pkt_mgr_stat_type type)
{
    switch (type)
    {
#define XX(_type, _name, _val) case _type: return stat->_val;
    PKT_MGR_STAT_MAP(XX)
#undef XX
    default:
        return 0;
    }
}

/******************************************************************************
 * packet manager rte
 ******************************************************************************/

static struct packet_manager_rte *packet_manager_rte_new()
{
    struct packet_manager_rte *pkt_mgr_rte = calloc(1, sizeof(struct packet_manager_rte));
    if (pkt_mgr_rte == NULL)
    {
        PACKET_MANAGER_LOG_ERROR("failed to allocate memory for packet_manager_rte");
        return NULL;
    }

    for (int i = 0; i < PACKET_QUEUE_MAX; i++)
    {
        TAILQ_INIT(&pkt_mgr_rte->queue[i]);
    }

    return pkt_mgr_rte;
}

static void packet_manager_rte_free(struct packet_manager_rte *pkt_mgr_rte)
{
    struct packet *pkt = NULL;

    if (pkt_mgr_rte)
    {
        for (int i = 0; i < PACKET_QUEUE_MAX; i++)
        {
            while ((pkt = TAILQ_FIRST(&pkt_mgr_rte->queue[i])))
            {
                TAILQ_REMOVE(&pkt_mgr_rte->queue[i], pkt, stage_tqe);
                packet_free(pkt);
            }
        }

        free(pkt_mgr_rte);
        pkt_mgr_rte = NULL;
    }
}

/******************************************************************************
 *  packet manager
 ******************************************************************************/

struct packet_manager *packet_manager_new(uint16_t thread_num)
{
    struct packet_manager *pkt_mgr = calloc(1, sizeof(struct packet_manager));
    if (pkt_mgr == NULL)
    {
        PACKET_MANAGER_LOG_ERROR("failed to allocate memory for packet_manager");
        return NULL;
    }

    pkt_mgr->thread_num = thread_num;
    pkt_mgr->ex_sche = exdata_schema_new();
    if (pkt_mgr->ex_sche == NULL)
    {
        PACKET_MANAGER_LOG_ERROR("failed to create exdata_schema");
        goto error_out;
    }

    pkt_mgr->fs = fieldstat_easy_new(pkt_mgr->thread_num, "packet_manager", NULL, 0);
    if (pkt_mgr->fs == NULL)
    {
        PACKET_MANAGER_LOG_ERROR("failed to create fieldstat_easy");
        goto error_out;
    }
    if (fieldstat_easy_enable_auto_output(pkt_mgr->fs, "metrics/packet_manager.json", 2) != 0)
    {
        PACKET_MANAGER_LOG_ERROR("failed to enable auto output for fieldstat_easy");
        goto error_out;
    }
    for (int i = 0; i < PKT_MGR_STAT_MAX; i++)
    {
        pkt_mgr->fs_idx[i] = fieldstat_easy_register_counter(pkt_mgr->fs, pkt_mgr_stat_str[i]);
    }

    return pkt_mgr;

error_out:
    packet_manager_free(pkt_mgr);
    return NULL;
}

void packet_manager_free(struct packet_manager *pkt_mgr)
{
    if (pkt_mgr)
    {
        if (pkt_mgr->fs)
        {
            fieldstat_easy_free(pkt_mgr->fs);
        }

        exdata_schema_free(pkt_mgr->ex_sche);

        free(pkt_mgr);
        pkt_mgr = NULL;
    }
}

int packet_manager_new_packet_exdata_index(struct packet_manager *pkt_mgr, const char *name, exdata_free *func, void *arg)
{
    assert(pkt_mgr);
    return exdata_schema_new_index(pkt_mgr->ex_sche, name, func, arg);
}

int packet_manager_register_node(struct packet_manager *pkt_mgr, const char *name, enum packet_stage stage,
                                 uint64_t interested_tag_key_bits,
                                 uint64_t interested_tag_val_bits,
                                 on_packet_callback *node_entry, struct module *mod)
{
    assert(pkt_mgr);
    assert(stage < PACKET_STAGE_MAX);
    assert(node_entry);

    struct node_array *nodes = &pkt_mgr->nodes[stage];
    if (nodes->used >= MAX_NODE_PER_STAGE)
    {
        PACKET_MANAGER_LOG_ERROR("exceed max node per stage %d", MAX_NODE_PER_STAGE);
        return -1;
    }

    struct node *node = &nodes->array[nodes->used];
    strncpy(node->name, name, sizeof(node->name));
    node->interested_tag_key_bits = interested_tag_key_bits;
    node->interested_tag_val_bits = interested_tag_val_bits;
    node->node_entry = node_entry;
    node->mod = mod;

    nodes->used++;
    return 0;
}

int packet_manager_init(struct packet_manager *pkt_mgr, uint16_t thread_id)
{
    assert(pkt_mgr);
    assert(thread_id < pkt_mgr->thread_num);

    pkt_mgr->rte[thread_id] = packet_manager_rte_new();
    if (pkt_mgr->rte[thread_id] == NULL)
    {
        PACKET_MANAGER_LOG_ERROR("failed to create packet_manager_rte");
        return -1;
    }
    else
    {
        return 0;
    }
}

void packet_manager_clean(struct packet_manager *pkt_mgr, uint16_t thread_id)
{
    assert(pkt_mgr);
    assert(thread_id < pkt_mgr->thread_num);

    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];
    packet_manager_rte_free(pkt_mgr_rte);
    pkt_mgr_rte = NULL;
}

void packet_manager_ingress(struct packet_manager *pkt_mgr, uint16_t thread_id, struct packet *pkt)
{
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];
    packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche));
    packet_tag_clean(pkt);

    pkt_mgr_rte->stat.pkts_ingress++;
    pkt_mgr_rte->stat.queue[PACKET_STAGE_PREROUTING].pkts_in++;
    TAILQ_INSERT_TAIL(&pkt_mgr_rte->queue[PACKET_STAGE_PREROUTING], pkt, stage_tqe);
}

struct packet *packet_manager_egress(struct packet_manager *pkt_mgr, uint16_t thread_id)
{
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];
    struct packet *pkt = TAILQ_FIRST(&pkt_mgr_rte->queue[PACKET_STAGE_MAX]);

    if (pkt)
    {
        pkt_mgr_rte->stat.pkts_egress++;
        pkt_mgr_rte->stat.queue[PACKET_STAGE_MAX].pkts_out++;
        TAILQ_REMOVE(&pkt_mgr_rte->queue[PACKET_STAGE_MAX], pkt, stage_tqe);
        exdata_runtime_free((struct exdata_runtime *)packet_get_user_data(pkt));

        return pkt;
    }
    else
    {
        return NULL;
    }
}

static void packet_tag_set_ip_proto(struct packet *pkt)
{
    switch (packet_get_ip_proto(pkt))
    {
    case IPPROTO_TCP:
        packet_tag_set(pkt, PKT_TAG_KEY_IPPROTO, PKT_TAG_VAL_IPPROTO_TCP);
        break;
    case IPPROTO_UDP:
        packet_tag_set(pkt, PKT_TAG_KEY_IPPROTO, PKT_TAG_VAL_IPPROTO_UDP);
        break;
    case IPPROTO_ICMP: /* fall through */
    case IPPROTO_ICMPV6:
        packet_tag_set(pkt, PKT_TAG_KEY_IPPROTO, PKT_TAG_VAL_IPPROTO_ICMP);
        break;
    default:
        break;
    }
}

void packet_manager_dispatch(struct packet_manager *pkt_mgr, uint16_t thread_id)
{
    uint64_t pkt_tag_key_bits = 0;
    uint64_t pkt_tag_val_bits = 0;
    struct packet *pkt = NULL;
    struct node *node = NULL;
    struct node_array *nodes = NULL;
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];

    for (int i = 0; i < PACKET_STAGE_MAX; i++)
    {
        pkt_mgr_rte->curr_stage = i;
        nodes = &pkt_mgr->nodes[pkt_mgr_rte->curr_stage];

        while ((pkt = TAILQ_FIRST(&pkt_mgr_rte->queue[pkt_mgr_rte->curr_stage])))
        {
            packet_set_stage(pkt, pkt_mgr_rte->curr_stage);
            packet_tag_set_ip_proto(pkt); // schedule packet may not set ip proto tag, so we need to set it here
            packet_set_claim(pkt, false);
            pkt_mgr_rte->claim_cb = NULL;
            pkt_mgr_rte->claim_arg = NULL;

            TAILQ_REMOVE(&pkt_mgr_rte->queue[pkt_mgr_rte->curr_stage], pkt, stage_tqe);
            pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_out++;

            for (uint16_t j = 0; j < nodes->used; j++)
            {
                node = &nodes->array[j];
                packet_tag_get(pkt, &pkt_tag_key_bits, &pkt_tag_val_bits); // pkt_tag may be changed by previous node, so we need to get it again
                if ((pkt_tag_key_bits & node->interested_tag_key_bits) &&
                    (pkt_tag_val_bits & node->interested_tag_val_bits))
                {
                    node->node_entry(pkt, node->mod);
                }
            }

            // packet has been claimed and cannot be released
            if (packet_is_claim(pkt))
            {
                if (pkt_mgr_rte->claim_cb)
                {
                    struct exdata_runtime *ex_rte = packet_get_user_data(pkt);
                    exdata_runtime_reset(ex_rte);

                    pkt_mgr_rte->claim_cb(pkt, pkt_mgr_rte->claim_arg);
                }
                continue;
            }

            if (packet_get_action(pkt) == PACKET_ACTION_DROP)
            {
                packet_manager_free_packet(pkt_mgr, thread_id, pkt);
                continue;
            }

            TAILQ_INSERT_TAIL(&pkt_mgr_rte->queue[pkt_mgr_rte->curr_stage + 1], pkt, stage_tqe);
            pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage + 1].pkts_in++;
        }
    }
    pkt_mgr_rte->curr_stage = PACKET_STAGE_MAX;
}

int packet_manager_claim_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, struct packet *pkt, on_packet_callback *cb, void *arg)
{
    assert(pkt_mgr);
    assert(thread_id < pkt_mgr->thread_num);
    assert(pkt);
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];

    if (packet_is_claim(pkt))
    {
        PACKET_MANAGER_LOG_ERROR("packet is already claimed, cannot claim again");
        return -1;
    }
    else
    {
        pkt_mgr_rte->claim_cb = cb;
        pkt_mgr_rte->claim_arg = arg;
        packet_set_claim(pkt, true);
        pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_claim++;
        return 0;
    }
}

void packet_manager_schedule_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, struct packet *pkt, enum packet_stage stage)
{
    assert(pkt_mgr);
    assert(thread_id < pkt_mgr->thread_num);
    assert(pkt);
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];

    if (stage >= PACKET_STAGE_MAX)
    {
        PACKET_MANAGER_LOG_ERROR("invalid stage %d", stage);
        assert(0);
        return;
    }

    pkt_mgr_rte->stat.queue[stage].pkts_schedule++;
    pkt_mgr_rte->stat.queue[stage].pkts_in++;
    TAILQ_INSERT_TAIL(&pkt_mgr_rte->queue[stage], pkt, stage_tqe);
}

struct packet_manager_stat *packet_manager_get_stat(struct packet_manager *pkt_mgr, uint16_t thread_id)
{
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];
    return &pkt_mgr_rte->stat;
}

void packet_manager_print_stat(struct packet_manager *pkt_mgr, uint16_t thread_id)
{
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];

    PACKET_MANAGER_LOG_INFO("runtime: %p, pkts_ingress: %lu, pkts_egress: %lu",
                            pkt_mgr_rte, pkt_mgr_rte->stat.pkts_ingress,
                            pkt_mgr_rte->stat.pkts_egress);
    for (int i = 0; i < PACKET_QUEUE_MAX; i++)
    {
        PACKET_MANAGER_LOG_INFO("runtime: %p, %-24s stat => "
                                "pkts_in: %lu, pkts_out: %lu, pkts_claim: %lu, pkts_schedule: %lu, pkts_drop: %lu, "
                                "pkts_dup_succ: %lu, pkts_dup_fail: %lu, "
                                "pkts_build_tcp_succ: %lu, pkts_build_tcp_fail: %lu, "
                                "pkts_build_udp_succ: %lu, pkts_build_udp_fail: %lu, "
                                "pkts_build_l3_succ: %lu, pkts_build_l3_fail: %lu",
                                pkt_mgr_rte,
                                packet_stage_to_str(i),
                                pkt_mgr_rte->stat.queue[i].pkts_in,
                                pkt_mgr_rte->stat.queue[i].pkts_out,
                                pkt_mgr_rte->stat.queue[i].pkts_claim,
                                pkt_mgr_rte->stat.queue[i].pkts_schedule,
                                pkt_mgr_rte->stat.queue[i].pkts_drop,
                                pkt_mgr_rte->stat.queue[i].pkts_dup_succ,
                                pkt_mgr_rte->stat.queue[i].pkts_dup_fail,
                                pkt_mgr_rte->stat.queue[i].pkts_build_tcp_succ,
                                pkt_mgr_rte->stat.queue[i].pkts_build_tcp_fail,
                                pkt_mgr_rte->stat.queue[i].pkts_build_udp_succ,
                                pkt_mgr_rte->stat.queue[i].pkts_build_udp_fail,
                                pkt_mgr_rte->stat.queue[i].pkts_build_l3_succ,
                                pkt_mgr_rte->stat.queue[i].pkts_build_l3_fail);
    }
}

struct packet *packet_manager_build_tcp_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, const struct packet *origin_pkt,
                                               uint32_t tcp_seq, uint32_t tcp_ack, uint8_t tcp_flags,
                                               const char *tcp_options, uint16_t tcp_options_len,
                                               const char *tcp_payload, uint16_t tcp_payload_len)
{
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];
    struct packet *pkt = packet_build_tcp(origin_pkt, tcp_seq, tcp_ack, tcp_flags, tcp_options, tcp_options_len, tcp_payload, tcp_payload_len);
    if (pkt == NULL)
    {
        pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_tcp_fail++;
        return NULL;
    }
    pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_tcp_succ++;
    packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche));
    packet_tag_clean(pkt);

    return pkt;
}

struct packet *packet_manager_build_udp_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, const struct packet *origin_pkt,
                                               const char *udp_payload, uint16_t udp_payload_len)
{
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];
    struct packet *pkt = packet_build_udp(origin_pkt, udp_payload, udp_payload_len);
    if (pkt == NULL)
    {
        pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_udp_fail++;
        return NULL;
    }
    pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_udp_succ++;
    packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche));
    packet_tag_clean(pkt);

    return pkt;
}

struct packet *packet_manager_build_l3_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, const struct packet *origin_pkt,
                                              uint8_t ip_proto, const char *l3_payload, uint16_t l3_payload_len)
{
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];
    struct packet *pkt = packet_build_l3(origin_pkt, ip_proto, l3_payload, l3_payload_len);
    if (pkt == NULL)
    {
        pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_l3_fail++;
        return NULL;
    }
    pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_l3_succ++;
    packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche));
    packet_tag_clean(pkt);

    return pkt;
}

struct packet *packet_manager_dup_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, const struct packet *origin_pkt)
{
    struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];
    struct packet *pkt = packet_dup(origin_pkt);
    if (pkt == NULL)
    {
        pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_dup_fail++;
        return NULL;
    }
    pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_dup_succ++;
    packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche));
    packet_tag_clean(pkt);

    return pkt;
}

void packet_manager_free_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, struct packet *pkt)
{
    if (pkt)
    {
        struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id];
        pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_drop++;
        exdata_runtime_free((struct exdata_runtime *)packet_get_user_data(pkt));
        packet_free(pkt);
    }
}

/******************************************************************************
 * packet manager module
 ******************************************************************************/

static void on_polling(struct module_manager *mod_mgr, void *args)
{
    uint64_t now_ms = clock_get_real_time_ms();
    int thread_id = module_manager_get_thread_id(mod_mgr);
    struct packet_manager *pkt_mgr = (struct packet_manager *)args;

    static __thread uint64_t last_sync_stat_ms = 0;
    static __thread struct packet_manager_stat pkt_mgr_last_stat = {0};
    if (now_ms - last_sync_stat_ms >= SYNC_STAT_INTERVAL_MS)
    {
        struct packet_manager_stat *pkt_mgr_curr_stat = &pkt_mgr->rte[thread_id]->stat;
        for (int i = 0; i < PKT_MGR_STAT_MAX; i++)
        {
            uint64_t val = packet_manager_stat_get(pkt_mgr_curr_stat, i) - packet_manager_stat_get(&pkt_mgr_last_stat, i);
            fieldstat_easy_counter_incrby(pkt_mgr->fs, thread_id, pkt_mgr->fs_idx[i], NULL, 0, val);
        }
        pkt_mgr_last_stat = *pkt_mgr_curr_stat;
        last_sync_stat_ms = now_ms;
    }
}

struct packet_manager *module_to_packet_manager(struct module *mod)
{
    assert(mod);
    assert(strcmp(module_get_name(mod), PACKET_MANAGER_MODULE_NAME) == 0);
    return (struct packet_manager *)module_get_ctx(mod);
}

struct module *packet_manager_on_init(struct module_manager *mod_mgr)
{
    assert(mod_mgr);
    uint16_t thread_num = module_manager_get_max_thread_num(mod_mgr);

    struct packet_manager *pkt_mgr = packet_manager_new(thread_num);
    if (pkt_mgr == NULL)
    {
        return NULL;
    }
    module_manager_register_polling_node(mod_mgr, on_polling, pkt_mgr);

    struct module *pkt_mgr_mod = module_new(PACKET_MANAGER_MODULE_NAME, NULL);
    if (pkt_mgr_mod == NULL)
    {
        PACKET_MANAGER_LOG_ERROR("failed to create packet_manager");
        packet_manager_free(pkt_mgr);
        return NULL;
    }
    module_set_ctx(pkt_mgr_mod, pkt_mgr);

    PACKET_MANAGER_LOG_FATAL("packet_manager init");
    return pkt_mgr_mod;
}

void packet_manager_on_exit(struct module_manager *mod_mgr __attribute__((unused)), struct module *mod)
{
    if (mod)
    {
        struct packet_manager *pkt_mgr = module_get_ctx(mod);

        packet_manager_free(pkt_mgr);
        module_free(mod);
        PACKET_MANAGER_LOG_FATAL("packet_manager exit");
    }
}

struct module *packet_manager_on_thread_init(struct module_manager *mod_mgr __attribute__((unused)), int thread_id, struct module *mod)
{
    struct packet_manager *pkt_mgr = module_get_ctx(mod);
    assert(pkt_mgr);
    assert(thread_id < pkt_mgr->thread_num);

    if (packet_manager_init(pkt_mgr, thread_id) != 0)
    {
        PACKET_MANAGER_LOG_ERROR("failed to init packet_manager_init");
        return NULL;
    }
    else
    {
        return mod;
    }
}

void packet_manager_on_thread_exit(struct module_manager *mod_mgr __attribute__((unused)), int thread_id, struct module *mod)
{
    struct packet_manager *pkt_mgr = module_get_ctx(mod);
    if (pkt_mgr)
    {
        assert(thread_id < pkt_mgr->thread_num);
        packet_manager_clean(pkt_mgr, thread_id);
    }
}