#pragma once #include #include #include #ifndef VNODE_CHECK_THREAD_SAFE #define VNODE_CHECK_THREAD_SAFE 0 #endif #ifndef VNODE_STAT_ENABLE #define VNODE_STAT_ENABLE 1 #endif struct shared_credict_counter { rte_spinlock_t lock; unsigned int counter; }; /* Tunnel Description Structure */ struct tunnel_desc { /* first cacheline, read only */ RTE_MARKER cacheline0; /* Tunel Object, real object to hold pointers */ struct rte_ring * tunnel_object; /* Tunnel Size */ unsigned int tunnel_size; unsigned int sz_en_buffer; struct rte_mbuf ** en_buffer; unsigned int sz_rt_buffer; struct rte_mbuf ** rt_buffer; /* second cache line, read/write on prod cores */ RTE_MARKER cacheline1 __rte_cache_min_aligned; /* Tunnel Enqueue Buffer Used */ unsigned int sz_en_buffer_used; /* Return Buffer Used */ unsigned int sz_rt_buffer_used; /* counters */ uint64_t on_line; uint64_t deliver; uint64_t missed; uint64_t total_len; /* queue length */ unsigned int q_len; float q_len_avg; RTE_MARKER cacheline2 __rte_cache_min_aligned; int32_t inflight_credits; #if VNODE_CHECK_THREAD_SAFE /* For debug, to check concurrent access of en_buffer */ rte_spinlock_t lock_thread_safe_check; #endif }; struct tunnel_block { /* Prod Queue Count */ unsigned int nr_prodq; /* Cons Queue Count */ unsigned int nr_consq; /* Tunnel's Producer */ struct vnode_prod * prod; /* Tunnel's Consumer */ struct vnode_cons * cons; /* Tunnel Descs */ struct tunnel_desc * descs[0]; }; struct vnode_mem_alloc_hints { unsigned int cpu_id_hints[MR_SID_MAX]; unsigned int numa_id_hints[MR_SID_MAX]; }; /* Virtual Data-Node Consumer Structure */ struct vnode_cons { TAILQ_ENTRY(vnode_cons) next; char symbol[MR_SYMBOL_MAX]; struct vnode * vnode; unsigned int nr_consq; struct tunnel_block * block; unsigned int cur_attach; struct vnode_cons_stat stat[MR_SID_MAX]; struct vnode_cons_notify notify[MR_SID_MAX]; }; /* Virtual Data-Node Producer Structure */ struct vnode_prod { TAILQ_ENTRY(vnode_prod) next; char symbol[MR_SYMBOL_MAX]; struct vnode * vnode; unsigned int nr_prodq; unsigned int cur_attach; struct tunnel_block * block; struct vnode_prod_stat stat[MR_SID_MAX]; }; TAILQ_HEAD(vnode_cons_list, vnode_cons); TAILQ_HEAD(vnode_prod_list, vnode_prod); /* Vitrual Data-Node Structure */ struct vnode { /* VNode Symbol */ char symbol[MR_SYMBOL_MAX]; /* Consumer */ struct vnode_cons * cons; /* Producer */ struct vnode_prod * prod; /* Tunnel Size */ unsigned int sz_tunnel; /* Tunnel Enqueue Buffer Size */ unsigned int sz_tunnel_buffer; /* shared credict */ unsigned int sz_shared; /* allow to notify cons when packet arrived. */ unsigned int notify_cons_when_rx; /* batch interval */ unsigned int batch_interval_tsc; /* q_len monitor */ unsigned int en_q_len_monitor; /* atomic, inflight credit */ int32_t credits_on_loan; int32_t max_inflight; int32_t max_credits_per_tunnel; /* Guarantees one operator(consumer or producer, create or destroy) a time */ rte_spinlock_t lock __rte_cache_aligned; }; #ifndef MR_LIBVNODE_MAX_SZ_BURST #define MR_LIBVNODE_MAX_SZ_BURST MR_BURST_MAX #endif #if VNODE_STAT_ENABLE #if VNODE_STAT_BY_ATOMIC #define VNODE_STAT_UPDATE(desc, queue, item, value) \ do \ { \ rte_atomic64_add(&desc->stat[queue].item, value); \ } while (0) #else #define VNODE_STAT_UPDATE(desc, queue, item, value) \ do \ { \ desc->stat[queue].item += value; \ } while (0) #endif #else #define VNODE_STAT_UPDATE(desc, queue, item, value) \ do \ { \ } while (0) #endif #define VNODE_STAT_UPDATE_AVG(desc, queue, item, value) \ do \ { \ desc->stat[queue].item += (typeof(desc->stat[queue].item))(0.2 * (value - desc->stat[queue].item)); \ } while (0) static inline struct tunnel_desc ** tunnel_block_locate(struct tunnel_block * block, unsigned int prodq_id, unsigned int consq_id) { assert(prodq_id < block->nr_prodq && consq_id < block->nr_consq); return &(block->descs[prodq_id * block->nr_consq + consq_id]); } /* VNode Structure Operation Functions */ struct vnode * __vnode_common_create(const char * sym, unsigned int sz_tunnel, unsigned int sz_tunnel_buffer, unsigned int notify_cons_when_rx); struct vnode_prod * __vnode_common_create_prod(struct vnode * vnode, const char * symbol, int nr_prodq); struct vnode_cons * __vnode_common_create_cons(struct vnode * vnode, const char * symbol, int nr_consq); struct vnode_prod * __vnode_common_prod_lookup(struct vnode * vnode, const char * sym); struct vnode_cons * __vnode_common_cons_lookup(struct vnode * vnode, const char * sym); int __vnode_common_cons_attach(struct vnode * node, struct vnode_cons * cons); int __vnode_common_prod_attach(struct vnode * node, struct vnode_prod * prod); struct vnode_prod_stat * __vnode_common_prod_stat_get(struct vnode_prod * prod); struct vnode_cons_stat * __vnode_common_cons_stat_get(struct vnode_cons * cons); int __vnode_common_delete_prod(struct vnode_prod * prod); int __vnode_common_delete_cons(struct vnode_cons * cons); int __vnode_common_delete(struct vnode * vnode); void __vnode_common_unpoison(struct vnode * vnode); void __vnode_common_unpoison_prod(struct vnode_prod * prod); void __vnode_common_unpoison_cons(struct vnode_cons * cons); struct vnode_cons_notify * __vnode_common_cons_notify_ctx(struct vnode_cons * cons); #define __USE_COMMON_VNODE_CREATE_PROD(_type) \ struct vnode_prod * vnode_##_type##_create_prod(struct vnode * vnode, const char * symbol, int nr_prodq) \ { \ return __vnode_common_create_prod(vnode, symbol, nr_prodq); \ } #define __USE_COMMON_VNODE_CREATE_CONS(_type) \ struct vnode_cons * vnode_##_type##_create_cons(struct vnode * vnode, const char * symbol, int nr_prodq) \ { \ return __vnode_common_create_cons(vnode, symbol, nr_prodq); \ } #define __USE_COMMON_VNODE_PROD_LOOKUP(_type) \ struct vnode_prod * vnode_##_type##_prod_lookup(struct vnode * vnode, const char * sym) \ { \ return __vnode_common_prod_lookup(vnode, sym); \ } #define __USE_COMMON_VNODE_CONS_LOOKUP(_type) \ struct vnode_cons * vnode_##_type##_cons_lookup(struct vnode * vnode, const char * sym) \ { \ return __vnode_common_cons_lookup(vnode, sym); \ } #define __USE_COMMON_VNODE_PROD_ATTACH(_type) \ int vnode_##_type##_prod_attach(struct vnode * node, struct vnode_prod * prod) \ { \ return __vnode_common_prod_attach(node, prod); \ } #define __USE_COMMON_VNODE_CONS_ATTACH(_type) \ int vnode_##_type##_cons_attach(struct vnode * node, struct vnode_cons * cons) \ { \ return __vnode_common_cons_attach(node, cons); \ } #define __USE_COMMON_VNODE_PROD_STAT_GET(_type) \ struct vnode_prod_stat * vnode_##_type##_prod_stat_get(struct vnode_prod * prod) \ { \ return __vnode_common_prod_stat_get(prod); \ } #define __USE_COMMON_VNODE_CONS_STAT_GET(_type) \ struct vnode_cons_stat * vnode_##_type##_cons_stat_get(struct vnode_cons * cons) \ { \ return __vnode_common_cons_stat_get(cons); \ } #define __USE_COMMON_VNODE_DELETE_PROD(_type) \ int vnode_##_type##_delete_prod(struct vnode_prod * prod) \ { \ return __vnode_common_delete_prod(prod); \ } #define __USE_COMMON_VNODE_DELETE_CONS(_type) \ int vnode_##_type##_delete_cons(struct vnode_cons * cons) \ { \ return __vnode_common_delete_cons(cons); \ } #define __USE_COMMON_VNODE_UNPOISON_PROD(_type) \ void vnode_##_type##_unpoison_prod(struct vnode_prod * prod) \ { \ return __vnode_common_unpoison_prod(prod); \ } #define __USE_COMMON_VNODE_UNPOISON_CONS(_type) \ void vnode_##_type##_unpoison_cons(struct vnode_cons * cons) \ { \ return __vnode_common_unpoison_cons(cons); \ } #define __USE_COMMON_VNODE_NOTIFY_CTX_CONS(_type) \ struct vnode_cons_notify * vnode_##_type##_notify_ctx_cons(struct vnode_cons * cons) \ { \ return __vnode_common_cons_notify_ctx(cons); \ }