#include "flowood.h" #include "flowood_fun.h" #include "flwd_net.h" #include #include "MESA_list_queue.h" #include "MESA_handle_logger.h" #include "MESA_prof_load.h" #include #include #include #include #include #include #include #include #include #ifdef __cplusplus /* 需要动态dlopen获取符号名, 使用C规范 */ extern "C" { #endif typedef struct { pthread_t pid; unsigned char thread_seq; int sd_raw_eth; /* 底层发包socket句柄 */ struct sockaddr sock_addr; MESA_lqueue_head pkt_queue; }pcap_work_thread_t; typedef struct{ char *__data_buf; /* mbuff缓冲区头指针 */ char *data_actual_ptr; /* 当前缓存实际存储的数据指针 */ int __data_buf_max_len; /* mbuff最大缓存长度, 即FLWD_LINK_MTU */ int data_actual_len; /* 当前缓存中实际存储的数据长度 */ }pcap_mbuff_t; /* pcap模式内部句柄结构 */ typedef struct{ int tot_thread_count; pcap_t *pcap_dev_handle; pcap_work_thread_t pcap_io_thread_para[FLWD_MAX_THREAD_NUM]; pcap_mbuff_t pcap_send_buf[FLWD_MAX_THREAD_NUM]; /* 发包缓存, 初始化时申请, 以后发包不用每次动态malloc/free */ }flwd_device_handle_pcap_t; typedef struct{ char *pkt_data; int pkt_len; }pcap_queue_item_t; /* 2012-04-10 LiJia add, 获取网卡MAC地址 参数: device: 网卡名称 mac: 存储MAC地址的数组,结果为网络序, 如网卡MAC地址为11:22:33:44:55:66,则mac[0]为0x11,mac[5]为0x66. 返回值: 0: 正常 -1:错误 */ static int MESA_get_dev_mac(const char *device, unsigned char mac[6]) { struct ifreq ifr; int fd; fd = socket(AF_INET, SOCK_DGRAM, 0); if(fd < 0) { return -1; } memset(ifr.ifr_ifrn.ifrn_name, 0, sizeof(ifr.ifr_ifrn.ifrn_name)); strncpy(ifr.ifr_ifrn.ifrn_name, device, sizeof(ifr.ifr_ifrn.ifrn_name)); if(ioctl(fd, SIOCGIFHWADDR, &ifr) == -1) { printf("Cann't get hwaddr of %s:%s\n", device, strerror(errno)); goto err_exit; } if(ifr.ifr_hwaddr.sa_family != ARPHRD_ETHER) { printf("'%s' is not ethernet interface!\n", device); goto err_exit; } memcpy(mac, ifr.ifr_ifru.ifru_addr.sa_data, 6); close(fd); return 0; err_exit: close(fd); return -1; } static inline int flwd_pcap_dispatch_thread(flwd_device_handle_pcap_t *pcap_io_handle, const u_char *data) { int to_work_thread_seq; const flwd_eth_hdr_t *flwd_eth_hdr = (flwd_eth_hdr_t *)data; const flwd_ipv4_hdr_t *flwd_ip_hdr; if(ETH_P_IP == ntohs(flwd_eth_hdr->h_proto)){ flwd_ip_hdr = (flwd_ipv4_hdr_t *)(data + sizeof(flwd_eth_hdr_t)); to_work_thread_seq = (flwd_ip_hdr->ip_src.s_addr ^ flwd_ip_hdr->ip_dst.s_addr) % pcap_io_handle->tot_thread_count; }else{ to_work_thread_seq = 0;/* 其他协议默认用0核心 */ } return to_work_thread_seq; } static pcap_mbuff_t *flwd_pcap_create_new_mbuff(u_char *data, int datalen) { pcap_mbuff_t *pcap_mbuff; pcap_mbuff = (pcap_mbuff_t *)calloc(1, sizeof(pcap_mbuff_t)); pcap_mbuff->__data_buf = (char *)malloc(datalen + FLWD_VXLAN_OUTER_PACKET_LEN); /* 预留vxlan头部 */ pcap_mbuff->__data_buf_max_len = datalen + FLWD_VXLAN_OUTER_PACKET_LEN; pcap_mbuff->data_actual_ptr = pcap_mbuff->__data_buf + FLWD_VXLAN_OUTER_PACKET_LEN; pcap_mbuff->data_actual_len = datalen; memcpy(pcap_mbuff->data_actual_ptr, data, datalen); return pcap_mbuff; } static void flwd_pcap_pkt_handle(u_char *user, const struct pcap_pkthdr *hdr, const u_char *data) { int to_work_thread_seq; pcap_mbuff_t *pcap_mbuff; flwd_device_handle_pcap_t *pcap_io_handle = (flwd_device_handle_pcap_t *)user; to_work_thread_seq = flwd_pcap_dispatch_thread(pcap_io_handle, data); pcap_mbuff = flwd_pcap_create_new_mbuff((u_char *)data, hdr->caplen); /* NOTE: 此处可以用阻塞接口, 因为pcap_loop在独立的线程上下文空间, 不影响其他流程 */ int ret = MESA_lqueue_join_tail(pcap_io_handle->pcap_io_thread_para[to_work_thread_seq].pkt_queue, &pcap_mbuff, sizeof(void *)); if(ret < 0){ free(pcap_mbuff->__data_buf); free(pcap_mbuff); printf("pcap io thread MESA_lqueue_join_tail() error, ret=%d\n", ret); } } static int pcap_low_level_io_init(flwd_device_handle_t *global_dev_handle) { int i; char pcap_errbuf[PCAP_ERRBUF_SIZE]; char str_tmp[128]; struct bpf_program bpf_filter; flwd_device_handle_pcap_t *pcap_io_handle = (flwd_device_handle_pcap_t *)calloc(1, sizeof(flwd_device_handle_pcap_t)); pcap_io_handle->pcap_dev_handle = pcap_open_live(global_dev_handle->io_para.device_name, 4096, 1, 1, pcap_errbuf); if(NULL == pcap_io_handle->pcap_dev_handle){ flwd_log(RLOG_LV_FATAL, "pcap_open_live() error, %s\n", pcap_errbuf); return -1; } /* TODO, set filter */ if(global_dev_handle->io_para.pkt_filter != NULL){ if(pcap_compile(pcap_io_handle->pcap_dev_handle, &bpf_filter, (char *)global_dev_handle->io_para.pkt_filter, 1, 0) < 0){ flwd_log(RLOG_LV_FATAL, "pcap_compile() error, invalid pkt_filter: %s\n", global_dev_handle->io_para.pkt_filter); return -1; } pcap_setfilter(pcap_io_handle->pcap_dev_handle, &bpf_filter); } pcap_setdirection(pcap_io_handle->pcap_dev_handle, PCAP_D_IN); /* 只捕收包 */ for(i = 0; i < global_dev_handle->tot_thread_count; i++){ pcap_io_handle->pcap_io_thread_para[i].thread_seq = i; pcap_io_handle->pcap_send_buf[i].__data_buf = (char *)malloc(FLWD_LINK_MTU); pcap_io_handle->pcap_send_buf[i].__data_buf_max_len = FLWD_LINK_MTU; pcap_io_handle->pcap_io_thread_para[i].sd_raw_eth = socket(PF_INET, SOCK_PACKET, htons(ETH_P_ALL)); snprintf(pcap_io_handle->pcap_io_thread_para[i].sock_addr.sa_data, sizeof(pcap_io_handle->pcap_io_thread_para[i].sock_addr.sa_data), "%s", global_dev_handle->io_para.device_name); pcap_io_handle->pcap_io_thread_para[i].pkt_queue = MESA_lqueue_create(1, 50000); } MESA_load_profile_string_def(FLWD_CONFIG_FILE, g_packet_io_cfg_para[(int)global_dev_handle->io_para.topo_mode].cfg_file_section, "addr_para", str_tmp, 128, "#"); if('#' == str_tmp[0]){ flwd_log(RLOG_LV_FATAL, "get config %s->addr_para error!", g_packet_io_cfg_para[(int)global_dev_handle->io_para.topo_mode].cfg_file_section); return -1; } if(inet_pton(AF_INET, str_tmp, &global_dev_handle->io_para.device_ip_net_order) <= 0){ flwd_log(RLOG_LV_FATAL, "config %s->addr_para type invalid!", g_packet_io_cfg_para[(int)global_dev_handle->io_para.topo_mode].cfg_file_section); return -1; } MESA_load_profile_string_def(FLWD_CONFIG_FILE, g_packet_io_cfg_para[(int)global_dev_handle->io_para.topo_mode].cfg_file_section, "addr_mask", str_tmp, 128, "#"); if('#' == str_tmp[0]){ flwd_log(RLOG_LV_FATAL, "get config %s->addr_mask error!", g_packet_io_cfg_para[(int)global_dev_handle->io_para.topo_mode].cfg_file_section); return -1; } if(inet_pton(AF_INET, str_tmp, &global_dev_handle->io_para.device_ip_mask_net_order) <= 0){ flwd_log(RLOG_LV_FATAL, "config %s->addr_mask type invalid!", g_packet_io_cfg_para[(int)global_dev_handle->io_para.topo_mode].cfg_file_section); return -1; } MESA_load_profile_string_def(FLWD_CONFIG_FILE, g_packet_io_cfg_para[(int)global_dev_handle->io_para.topo_mode].cfg_file_section, "gateway_ip", str_tmp, 128, "#"); if('#' == str_tmp[0]){ flwd_log(RLOG_LV_FATAL, "get config %s->gateway_ip error!", g_packet_io_cfg_para[(int)global_dev_handle->io_para.topo_mode].cfg_file_section); return -1; } if(inet_pton(AF_INET, str_tmp, &global_dev_handle->io_para.gateway_ip_net_order) <= 0){ flwd_log(RLOG_LV_FATAL, "config %s->gateway_ip type invalid!", g_packet_io_cfg_para[(int)global_dev_handle->io_para.topo_mode].cfg_file_section); return -1; } MESA_get_dev_mac(global_dev_handle->io_para.device_name, global_dev_handle->io_para.local_mac_addr); pcap_io_handle->tot_thread_count = global_dev_handle->tot_thread_count; global_dev_handle->low_level_io_handle = (void *)pcap_io_handle; return 0; } static int pcap_low_level_send(flwd_device_handle_t *global_dev_handle, int tid, void *mbuff) { int ret; pcap_mbuff_t *pcap_mbuff = (pcap_mbuff_t *)mbuff; flwd_device_handle_pcap_t *pcap_io_handle = (flwd_device_handle_pcap_t *)global_dev_handle->low_level_io_handle; ret = sendto(pcap_io_handle->pcap_io_thread_para[tid].sd_raw_eth, pcap_mbuff->data_actual_ptr, pcap_mbuff->data_actual_len, MSG_DONTWAIT, (const struct sockaddr *)&pcap_io_handle->pcap_io_thread_para[tid].sock_addr, sizeof(struct sockaddr)); return ret; } int low_level_pkt_recv(flwd_device_handle_t *g_dev_handle, int tid, void **mbuff) { int ret, try_times; pcap_mbuff_t *pcap_mbuff; long recv_len; flwd_device_handle_pcap_t *pcap_io_handle = (flwd_device_handle_pcap_t *)g_dev_handle->low_level_io_handle; recv_len = sizeof(void *); for(try_times = 0; try_times < 20; try_times++){ /* NOTE: 因为是trylock, 非常容易失败, 尝试连续多试几次 */ ret = MESA_lqueue_try_get_head(pcap_io_handle->pcap_io_thread_para[tid].pkt_queue, &pcap_mbuff, &recv_len); switch(ret){ case 0: //OK goto done; break; case -5: //empty, no packet return -1; } } if(ret < 0){ return -1; } done: *mbuff = (void *)pcap_mbuff; return pcap_mbuff->data_actual_len; } void low_level_pkt_free(flwd_device_handle_t *g_dev_handle, int tid, void *mbuff) { pcap_mbuff_t *pcap_mbuff = (pcap_mbuff_t *)mbuff; free(pcap_mbuff->__data_buf); free(pcap_mbuff); return; } void *low_level_mbuff_malloc(flwd_device_handle_t *g_dev_handle, int tid, int len) { ///flwd_device_handle_pcap_t *pcap_io_handle = (flwd_device_handle_pcap_t *)g_dev_handle->low_level_io_handle; pcap_mbuff_t *mbuff = (pcap_mbuff_t *)malloc(sizeof(pcap_mbuff_t)); mbuff->__data_buf = (char *)malloc(FLWD_LINK_MTU); mbuff->data_actual_ptr = mbuff->__data_buf + FLWD_VXLAN_OUTER_PACKET_LEN; mbuff->__data_buf_max_len = FLWD_LINK_MTU; mbuff->data_actual_len = 0; return (void *)mbuff; } char *low_level_mbuff_mtod(void *mbuff) { pcap_mbuff_t *pcap_mbuff = (pcap_mbuff_t *)mbuff; return (char *)pcap_mbuff->data_actual_ptr; } void low_level_mbuff_free(struct __flwd_device_handle *h, int tid, void *mbuff) { pcap_mbuff_t *pcap_mbuff = (pcap_mbuff_t *)mbuff; free(pcap_mbuff->__data_buf); pcap_mbuff->data_actual_len = 0; pcap_mbuff->__data_buf_max_len = 0; free(mbuff); } void low_level_mbuff_free_after_send(flwd_device_handle_t *g_dev_handle, int tid, void *mbuff) { /* pcap模式发送无自动free, 再次调用真正的free函数 */ low_level_mbuff_free(g_dev_handle, tid, mbuff); } void low_level_mbuff_send_back(flwd_device_handle_t *g_dev_handle, int tid, void *mbuff) { /* pcap模式暂时不用回注gdev数据包, 空函数 */ return; } /* 在mbuff里追加user_data数据. return value: 数据起始位置; */ char * low_level_mbuff_data_append(void *mbuff, const char *user_data, int user_data_len) { pcap_mbuff_t *pcap_mbuff= (pcap_mbuff_t *)mbuff; assert(user_data_len <= pcap_mbuff->__data_buf_max_len - pcap_mbuff->data_actual_len); memcpy(pcap_mbuff->data_actual_ptr, user_data, user_data_len); pcap_mbuff->data_actual_len += user_data_len; return (char *)pcap_mbuff->data_actual_ptr; } /* 数据指针向前移动N个字节, 数据长度自动增加N, 功能是在现有数据包之前填充其他包头, 如vxlan */ char *low_level_mbuff_data_forward(void *mbuff, int n) { pcap_mbuff_t *pcap_mbuff = (pcap_mbuff_t *)mbuff; assert(n <= pcap_mbuff->data_actual_ptr - pcap_mbuff->__data_buf); pcap_mbuff->data_actual_ptr -= n; /* 先前移动 */ pcap_mbuff->data_actual_len += n; return (char *)pcap_mbuff->data_actual_ptr; } /* 数据指针向后移动N个字节, 数据长度自动减少N, 功能是把现有数据包的某些包头跳过, 如剥去vxlan头部 */ char * low_level_mbuff_data_rearward(void *mbuff, int n) { pcap_mbuff_t *pcap_mbuff = (pcap_mbuff_t *)mbuff; assert(n <= pcap_mbuff->__data_buf_max_len - pcap_mbuff->data_actual_len); pcap_mbuff->data_actual_ptr += n; pcap_mbuff->data_actual_len -= n; return pcap_mbuff->data_actual_ptr; } int low_level_mbuff_get_pkt_len(void *mbuff) { pcap_mbuff_t *pcap_mbuff = (pcap_mbuff_t *)mbuff; return pcap_mbuff->data_actual_len; } void low_level_mbuff_set_pkt_len(void *mbuff, int pkt_len) { pcap_mbuff_t *pcap_mbuff = (pcap_mbuff_t *)mbuff; pcap_mbuff->data_actual_len = pkt_len; } int low_level_send(flwd_device_handle_t *global_dev_handle, int tid, void *mbuff) { return pcap_low_level_send(global_dev_handle, tid, mbuff); } /* 此函数可以阻塞或死循环, 一直不返回 */ void low_level_io_run(flwd_device_handle_t *global_io_handle) { ///int i; ///pthread_t pid; flwd_device_handle_pcap_t *pcap_io_handle = (flwd_device_handle_pcap_t *)global_io_handle->low_level_io_handle; while(1){ pcap_loop(pcap_io_handle->pcap_dev_handle, -1, flwd_pcap_pkt_handle, (u_char *)pcap_io_handle); } return; } int low_level_io_init(flwd_device_handle_t *global_dev_handle) { return pcap_low_level_io_init(global_dev_handle); } #ifdef __cplusplus } #endif