增加BPFDUMP功能，支持在应用层面通过TAP虚拟设备导出报文。该功能为调试功能，通过配置文件启用。v4.2.4-20170717

9 files changed, 380 insertions, 7 deletions

diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt
index 0f8b0a4..e61a7d8 100644
--- a/app/CMakeLists.txt
+++ b/app/CMakeLists.txt
@@ -7,7 +7,8 @@ include_directories(${DPDK_INCLUDE_DIR})
 
 include_directories(include)
 add_definitions(${DPDK_C_PREDEFINED})
-add_library(marsio SHARED src/marsio.c src/arp.c src/icmp.c src/neigh.c src/rawio.c src/mrb.c src/sendpath.c src/monit.c)
+add_library(marsio SHARED src/marsio.c src/arp.c src/icmp.c src/neigh.c src/rawio.c src/mrb.c 
+						  src/sendpath.c src/monit.c src/bpfdump.c)
 
 set_target_properties(marsio PROPERTIES VERSION ${MARSIO_VERSION_MAJOR}.${MARSIO_VERSION_MINOR})
 set_target_properties(marsio PROPERTIES SOVERSION ${MARSIO_VERSION_MAJOR})
diff --git a/app/include/bpfdump.h b/app/include/bpfdump.h
new file mode 100644
index 0000000..875030d
--- /dev/null
+++ b/app/include/bpfdump.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <rte_mbuf.h>
+
+struct bpf_dumper;
+
+enum bpf_dumper_backend
+{
+	BPF_DUMPER_TAP,
+	BPF_DUMPER_PCAP,
+	BPF_DUMPER_SYSLOG,
+	BPF_DUMPER_COUNT,
+	BPF_DUMPER_MAX
+};
+
+#define BPF_DUMPER_DEFAULT_BACKEND	BPF_DUMPER_TAP
+
+int bpf_dumper_write(struct bpf_dumper * dumper, struct rte_mbuf * __mbufs[],
+	unsigned int nr_mbufs);
+
+void bpf_dumper_destory(struct bpf_dumper * dumper);
+
+struct bpf_dumper * bpf_dumper_create(const char * appsym, enum bpf_dumper_backend backend,
+	const char * str_devsym, const char * str_path, const char * str_bpf_expr);
\ No newline at end of file
diff --git a/app/include/mrapp.h b/app/include/mrapp.h
index 6aad930..f070761 100644
--- a/app/include/mrapp.h
+++ b/app/include/mrapp.h
@@ -6,6 +6,8 @@
 #include <neigh.h>
 #include <marsio.h>
 #include <ldbc.h>
+#include <pcap/pcap.h>
+#include <bpfdump.h>
 
 struct mr_instance;
 
@@ -30,6 +32,8 @@ struct mrapp_stat
     uint64_t packet_send_drop;
 };
 
+
+
 /* 用户设备描述符 */
 struct mr_vdev
 {
@@ -40,6 +44,8 @@ struct mr_vdev
     unsigned int nr_txstream;
     unsigned int en_arp;
     unsigned int en_icmp;
+
+	struct bpf_dumper * bpf_dumper;
 };
 
 struct mr_thread_info
@@ -59,7 +65,9 @@ struct mr_instance
 	char monit_file_path[MR_STRING_MAX];
     /* 全局配置文件路径 */
     char g_cfgfile_path[MR_STRING_MAX];
-    /* 消息框架句柄 */
+	/* 应用配置文件路径 */
+	char app_cfgfile_path[MR_STRING_MAX];
+	/* 消息框架句柄 */
     struct ctrlmsg_handler * ctrlmsg_handler;
     /* 虚设备实例列表 */
     struct mr_vdev vdevs[MR_VDEV_MAX];
diff --git a/app/include/sendpath.h b/app/include/sendpath.h
index 03c17f6..df7f43b 100644
--- a/app/include/sendpath.h
+++ b/app/include/sendpath.h
@@ -44,6 +44,8 @@ struct mr_sendpath
     struct vdev_instance * target_vdi;
     /* Instance */
     struct mr_instance * instance;
+	/* __VDEV */
+	struct mr_vdev * vdev;
 };
 
 struct __mr_sendpath_vdev
diff --git a/app/src/bpfdump.c b/app/src/bpfdump.c
new file mode 100644
index 0000000..8452bdc
--- /dev/null
+++ b/app/src/bpfdump.c
@@ -0,0 +1,271 @@
+#include <common.h>
+#include <pcap/pcap.h>
+#include <rte_atomic.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+
+#include <netinet/in.h>
+#include <linux/if.h>
+#include <linux/if_tun.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+#include <signal.h>
+#include <assert.h>
+#include <bpfdump.h>
+
+static int __tap_count = 0;
+
+static const char * str_bpf_dumper_backend[] =
+{
+	[BPF_DUMPER_TAP] = "TAP",
+	[BPF_DUMPER_PCAP] = "PCAP",
+	[BPF_DUMPER_SYSLOG] = "SYSLOG",
+	[BPF_DUMPER_COUNT] = "COUNT"
+};
+
+struct __backend_ops
+{
+	int(*fn_create)(struct bpf_dumper * object, const char * appsym,
+		const char * str_devsym, const char * str_path);
+	void(*fn_destory)(struct bpf_dumper * object);
+	int(*fn_write)(struct bpf_dumper * object, const char * pkt_ptr, unsigned int pkt_len);
+};
+
+struct bpf_dumper
+{
+	char str_dumper_sym[MR_SYMBOL_MAX];
+	char str_bpf_expr[MR_STRING_MAX];
+	char str_dumpfile[MR_STRING_MAX];
+
+	struct bpf_program bpf_program;
+
+	unsigned int en_read;
+	unsigned int en_write;
+
+	enum bpf_dumper_backend backend;
+	struct __backend_ops * backend_ops;
+
+	struct __backend_tap
+	{
+		int tapdev_fd;
+	} backend_tap;
+
+	int bpf_offset;
+
+	rte_atomic64_t stat_write_pkts;
+	rte_atomic64_t stat_write_pktlen;
+	rte_atomic64_t stat_write_drops;
+	
+	rte_atomic64_t stat_read_pkts;
+	rte_atomic64_t stat_read_pktlen;
+	rte_atomic64_t stat_read_drops;
+};
+
+static int tap_ioctl(int fd, unsigned long request, struct ifreq *ifr, int set)

+{

+	short req_flags = ifr->ifr_flags;

+	switch (request) {

+	case SIOCSIFFLAGS:

+		if (ioctl(fd, SIOCGIFFLAGS, ifr) < 0)

+			goto error;

+		if (set)

+			ifr->ifr_flags |= req_flags;

+		else

+			ifr->ifr_flags &= ~req_flags;

+		break;

+

+	case SIOCGIFFLAGS:

+	case SIOCGIFHWADDR:

+	case SIOCSIFHWADDR:

+	case SIOCSIFMTU:

+		break;

+	default:

+		return -EINVAL;

+	}

+

+	if (ioctl(fd, request, ifr) < 0)

+		goto error;

+

+	return 0;

+

+error:

+	MR_ERROR("%s: ioctl(%lu) failed with error: %s", ifr->ifr_name, request, strerror(errno));

+	return -errno;

+}
+
+static int __tap_dumper_create(struct bpf_dumper * object, const char * appsym,
+	const char * str_devsym, const char * str_path)
+{
+	/* ����TUNTAP�豸ʹ�õ�FD */
+	int fd = 0;
+	int ioctl_sock = 0;
+
+	fd = open("/dev/net/tun", O_RDWR);
+	if (fd < 0)
+	{
+		MR_ERROR("Tap dumper %s open /dev/net/tun failed: %s", 
+			object->str_dumper_sym, strerror(errno)); goto __fail_out;
+	}
+
+	/* ����TUNTAP�豸ʱʹ�õ�FD */
+	ioctl_sock = socket(AF_INET, SOCK_DGRAM, 0);
+	if (ioctl_sock < 0)
+	{
+		MR_ERROR("Tap dumper %s unable to get a socket for management: %s", 
+			object->str_dumper_sym,	strerror(errno)); goto __fail_out;
+	}
+
+	struct ifreq ifr;
+	memset(&ifr, 0, sizeof(ifr));
+
+	snprintf(object->str_dumpfile, IFNAMSIZ, "%s_t%d", appsym, __tap_count++);
+
+	/* TAP device without packet information */
+	ifr.ifr_flags = IFF_TAP | IFF_NO_PI;
+	snprintf(ifr.ifr_name, IFNAMSIZ, "%s", object->str_dumpfile);
+		
+	int ret = ioctl(fd, TUNSETIFF, (void *)&ifr);
+	if (ret < 0)
+	{
+		MR_ERROR("Tap dumper %s tap fd(fd = %d) ioctl failed: %s", 
+			object->str_dumper_sym, fd, strerror(errno)); goto __fail_out;
+	}
+
+	struct ifreq link_up_ifr = { .ifr_flags = IFF_UP | IFF_RUNNING };
+	snprintf(link_up_ifr.ifr_name, IFNAMSIZ, "%s", object->str_dumpfile);
+
+	ret = tap_ioctl(ioctl_sock, SIOCSIFFLAGS, &link_up_ifr, 1);
+	if(ret < 0)
+	{
+		MR_ERROR("Tap dumper %s tap device link up failed", object->str_dumper_sym);
+		goto __fail_out;
+	}
+
+	MR_DEBUG("TAP dumper %s: tap fd = %d, ifr_name = %s",
+		object->str_dumper_sym, fd, ifr.ifr_name);
+
+	object->backend_tap.tapdev_fd = fd;
+	close(ioctl_sock);
+
+	return RT_SUCCESS;
+
+__fail_out:
+	if(fd > 0) close(fd);
+	if (ioctl_sock > 0) close(ioctl_sock);
+	return RT_ERR;
+}
+
+static void __tap_dumper_destory(struct bpf_dumper * object)
+{
+	close(object->backend_tap.tapdev_fd);
+	return;
+}
+
+static int __tap_dumper_write(struct bpf_dumper * object, const char * pkt_ptr, unsigned int pkt_len)
+{
+	return write(object->backend_tap.tapdev_fd, pkt_ptr, pkt_len);
+}
+
+static struct __backend_ops __backend_ops_jumper[] =
+{
+	[BPF_DUMPER_TAP] =
+	{
+		.fn_create = __tap_dumper_create,
+		.fn_destory = __tap_dumper_destory,
+		.fn_write = __tap_dumper_write
+	}
+};
+
+int bpf_dumper_write(struct bpf_dumper * dumper, struct rte_mbuf * __mbufs[], 
+	unsigned int nr_mbufs)
+{
+	unsigned int nr_write_success = 0;
+
+	for (unsigned int i = 0; i < nr_mbufs; i++)
+	{
+		struct rte_mbuf * __mbuf = __mbufs[i];
+
+		const char * pkt_ptr = rte_pktmbuf_mtod_offset(__mbuf, const char *, dumper->bpf_offset);
+		unsigned int pkt_len = rte_pktmbuf_data_len(__mbuf) > dumper->bpf_offset ?
+			rte_pktmbuf_data_len(__mbuf) - dumper->bpf_offset : 0;
+		
+		if (pkt_ptr == NULL || pkt_len == 0) goto __dropme;
+
+		if (bpf_filter(dumper->bpf_program.bf_insns,
+			(const u_char *)pkt_ptr, pkt_len, pkt_len) == 0 || pkt_len > 65535) goto __dropme;
+
+		assert(dumper->backend_ops->fn_write != NULL);
+		int ret = dumper->backend_ops->fn_write(dumper, pkt_ptr, pkt_len);
+		if (unlikely(ret < 0)) goto __dropme;
+
+		/* д��TAP�豸�ɹ� */
+		rte_atomic64_add(&dumper->stat_write_pkts, 1);
+		rte_atomic64_add(&dumper->stat_write_pktlen, pkt_len);
+		nr_write_success++;
+		continue;
+
+	__dropme:
+		rte_atomic64_add(&dumper->stat_write_drops, 1);
+		continue;
+	}
+
+	return nr_write_success;
+}
+
+void bpf_dumper_destory(struct bpf_dumper * dumper)
+{
+	if (dumper != NULL) dumper->backend_ops->fn_destory(dumper);
+	pcap_freecode(&dumper->bpf_program);
+	rte_free(dumper);
+	return;
+}
+
+struct bpf_dumper * bpf_dumper_create(const char * appsym, enum bpf_dumper_backend backend,
+	const char * str_devsym, const char * str_path, const char * str_bpf_expr)
+{
+	assert(backend >= 0 && backend < BPF_DUMPER_MAX);
+
+	struct bpf_dumper * dumper = rte_zmalloc(NULL, sizeof(struct bpf_dumper), 0);
+	MR_VERIFY_MALLOC(dumper);
+
+	snprintf(dumper->str_dumper_sym, sizeof(dumper->str_dumper_sym), "%s_%s_%s", 
+		appsym, str_devsym, str_bpf_dumper_backend[backend]);
+
+	dumper->backend = backend;
+	dumper->backend_ops = &__backend_ops_jumper[backend];
+	
+	/* Backend Init */
+	int ret = dumper->backend_ops->fn_create(dumper, appsym, str_devsym, str_path);
+	if (ret < 0)
+	{
+		MR_ERROR("BPF dumper %s backend create failed. ", dumper->str_dumper_sym);
+		goto __errout;
+	}
+
+	/* BPF Filter */
+	strncpy(dumper->str_bpf_expr, str_bpf_expr, sizeof(dumper->str_bpf_expr));
+
+	ret = pcap_compile_nopcap(65535, DLT_EN10MB, &dumper->bpf_program,
+		dumper->str_bpf_expr, 1, PCAP_NETMASK_UNKNOWN);
+
+	if (ret < 0)
+	{
+		MR_ERROR("BPF dumper %s bpf filter compile failed, bpf expr is '%s': %s",
+			dumper->str_dumper_sym, dumper->str_bpf_expr, pcap_strerror(ret));
+		goto __errout;
+	}
+
+	MR_INFO(" ");
+	MR_INFO("Application %s, BPF Dumper %s:", appsym, dumper->str_dumper_sym);
+	MR_INFO("   BPF expression                 : %s", dumper->str_bpf_expr);
+	MR_INFO("   BPF offset                     : %d", dumper->bpf_offset);
+	MR_INFO("   Backend                        : %s", str_bpf_dumper_backend[dumper->backend]);
+	MR_INFO("   Dumpfile(or device)            : %s", dumper->str_dumpfile);
+		
+	return dumper;
+	
+__errout:
+	if (dumper != NULL) bpf_dumper_destory(dumper);
+	return NULL;
+}
\ No newline at end of file
diff --git a/app/src/marsio.c b/app/src/marsio.c
index a77dc5f..7a82ac6 100644
--- a/app/src/marsio.c
+++ b/app/src/marsio.c
@@ -14,6 +14,7 @@
 #include <unistd.h>
 #include <signal.h>
 #include <cJSON.h>
+#include <libgen.h>
 
 #define MR_LIB_MAX_EAL_ARGC			128
 
@@ -175,6 +176,51 @@ static int mrapp_distributer_init(struct mr_instance * instance)
 	return RT_SUCCESS;
 }
 
+static int mrapp_bpf_dumper_init(struct mr_instance * instance, struct mr_vdev * vdev)
+{
+	const char * str_devsym = vdev->devsym;
+	const char * str_cfgfile = instance->app_cfgfile_path;
+	
+	/* 从应用配置文件中读取Dumper的配置。Dumper系调试使用，由用户通过配置文件指定，
+	   读不到相关选项，则不启用Dumper。 */
+
+	char str_section[MR_SYMBOL_MAX] = { 0 };
+	snprintf(str_section, sizeof(str_section), "bpfdump:%s", str_devsym);
+	
+	unsigned int __opt_enable = 0;
+	MESA_load_profile_uint_def(str_cfgfile, str_section, "enable", &__opt_enable, 0);
+
+	/* 是否启用 */
+	if (!__opt_enable) return RT_SUCCESS;
+
+	char __opt_str_bpf_expr[MR_STRING_MAX] = { 0 };
+	char __opt_str_dumpfile[MR_STRING_MAX] = { 0 };
+
+	unsigned int __opt_direction = 0;
+	unsigned int __opt_backend = 0;
+	unsigned int __opt_offset = 0;
+
+	MESA_load_profile_string_def(str_cfgfile, str_section, "bpf_expr", __opt_str_bpf_expr,
+		sizeof(__opt_str_bpf_expr), NULL);
+	MESA_load_profile_string_def(str_cfgfile, str_section, "dumpfile", __opt_str_dumpfile,
+		sizeof(__opt_str_dumpfile), NULL);
+
+	MESA_load_profile_uint_def(str_cfgfile, str_section, "bpf_offset", &__opt_offset, 0);
+	MESA_load_profile_uint_def(str_cfgfile, str_section, "direction", &__opt_direction, 0);
+	MESA_load_profile_uint_def(str_cfgfile, str_section, "backend", &__opt_backend, 0);
+
+	vdev->bpf_dumper = bpf_dumper_create(instance->appsym, __opt_backend,
+		str_devsym, __opt_str_dumpfile, __opt_str_bpf_expr);
+
+	if (unlikely(vdev->bpf_dumper == NULL))
+	{
+		MR_ERROR("In application %s, BPF dumper for device %s create failed.",
+			instance->appsym, str_devsym); return RT_ERR;
+	}
+
+	return RT_SUCCESS;
+}
+
 /* 注册应用 */
 static int mrapp_app_register(struct mr_instance * instance)
 {
@@ -234,7 +280,16 @@ static int mrapp_gconf_init(struct mr_instance * instance)
     char * g_cfg_file = cJSON_GetObjectItem(j_genernal, "g_cfgfile")->valuestring;
     if (g_cfg_file == NULL) goto j_parse_error;
     
+	/* 全局配置文件路径 */
     strncpy(instance->g_cfgfile_path, g_cfg_file, sizeof(instance->g_cfgfile_path));
+
+	/* 本地配置文件路径 */
+	char __str_cfgfile[MR_STRING_MAX] = { 0 };
+	strncpy(__str_cfgfile, instance->g_cfgfile_path, sizeof(__str_cfgfile));
+
+	snprintf(instance->app_cfgfile_path, sizeof(instance->app_cfgfile_path),
+		"%s/mrapp.%s.conf", dirname(__str_cfgfile), instance->appsym);
+
     return RT_SUCCESS;
 
 j_parse_error:
@@ -372,6 +427,8 @@ struct mr_vdev * marsio_open_device(struct mr_instance * instance,
     MR_INFO("   ARP protocol handler           : %s", vdev->en_arp ? "Enable" : "Disable");
     MR_INFO("   ICMP protocol handler          : %s", vdev->en_arp ? "Enable" : "Disable");
 
+	/* 调试捕包 */
+	mrapp_bpf_dumper_init(instance, vdev);
     return vdev;
 }
 
diff --git a/app/src/rawio.c b/app/src/rawio.c
index a434238..d8f92ca 100644
--- a/app/src/rawio.c
+++ b/app/src/rawio.c
@@ -67,6 +67,10 @@ int marsio_recv_burst(struct mr_vdev * vdev, queue_id_t qid, marsio_buff_t * mbu
         thread_info.instance->stat[tid].packet_recv_length = __packet_total_len(__mbufs, ret);
     }
 
+	/* BPF Dumper */
+	if (unlikely(vdev->bpf_dumper != NULL))
+		bpf_dumper_write(vdev->bpf_dumper, __mbufs, ret);
+
 out:
     return ret;
 }
@@ -124,7 +128,11 @@ int marsio_send_burst(struct mr_sendpath * sendpath, queue_id_t sid, marsio_buff
     /* 从报文本身携带的Hash值计算分流用的Hash值 */
     hash_t hash[MR_BURST_MAX];
     for (int i = 0; i < nr_mbufs; i++) hash[i] = __mbufs[i]->hash.usr;
-    
+
+	/* BPF Dumper */
+	if (unlikely(sendpath->vdev->bpf_dumper != NULL))
+		bpf_dumper_write(sendpath->vdev->bpf_dumper, __mbufs, nr_mbufs);
+
     vnode_mirror_enqueue_bulk(sendpath->target_vdi->vnode_tx_prod, sid, __mbufs, hash, nr_mbufs);
 
     /* 线程运行情况统计 */
diff --git a/app/src/sendpath.c b/app/src/sendpath.c
index d7991a5..8702cc6 100644
--- a/app/src/sendpath.c
+++ b/app/src/sendpath.c
@@ -86,6 +86,7 @@ static struct mr_sendpath * sendpath_vdev_create(struct mr_instance * instance,
 
 	/* Parameters */
 	_sendpath->_father.target_vdi = dest_vdev->vdi;
+	_sendpath->_father.vdev = dest_vdev;
     _sendpath->_father.can_use = 1;
 	
 	/* Callback Functions */
@@ -292,6 +293,7 @@ _build:
     /* 填充SendPath各虚函数指针*/
     sendpath->instance = target_vdev->instance;
     sendpath->target_vdi = target_vdev->vdi;
+	sendpath->vdev = target_vdev;
 
     sendpath->fn_requery = sendpath_route_requery;
     sendpath->fn_l2_construct = sendpath_route_l2_construct;
diff --git a/infra/include/ldbc.h b/infra/include/ldbc.h
index 72d4586..bdad3f2 100644
--- a/infra/include/ldbc.h
+++ b/infra/include/ldbc.h
@@ -118,10 +118,10 @@ static inline void pkt_parser_init(struct pkt_parser * pkt_parser,
 
 static inline int pkt_parser_push(struct pkt_parser * pkt_parser,
 	enum complex_layer this_layer_type, const char * data)
-{

-	assert(pkt_parser->nr_results < pkt_parser->nr_expect_results);

-	assert(pkt_parser->nr_results < RTE_DIM(pkt_parser->results));

-

+{
+	assert(pkt_parser->nr_results < pkt_parser->nr_expect_results);
+	assert(pkt_parser->nr_results < RTE_DIM(pkt_parser->results));
+
 	pkt_parser->results[pkt_parser->nr_results].data = data;
 	pkt_parser->results[pkt_parser->nr_results].this_layer_type = this_layer_type;
 	pkt_parser->nr_results++;