////////////////////////////////////////
///@file tx_sample.c
///
///@brief pag接口发包示例代码，可以指定发包的长度，类型，源目ip
///
///修改记录:
///
///@author lifengw@sugon.com
///@date 2013-10-13
///
///建立文档
////////////////////////////////////////

//系统路径下头文件
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>  //getopt
#include <string.h>
#include <pthread.h>
#include <signal.h>
#include <sys/time.h>
#include <linux/types.h>
#include <net/ethernet.h>
#include <netinet/ip.h>  //struct ip
#include <netinet/in.h>  //IPPROTO_TCP
#include <netinet/ip6.h>
#include <arpa/inet.h>  //inet_addr();

#include <libpag.h>

///// 宏定义 ///////

#define DEF_THREADS_NUM 16   ///<默认线程数
#define MAX_THREADS_NUM 64  ///<系统支持的最大线程数 

#define MIN_TX_PKT_LEN  60  ///<可发送的最小报文长度
#define MAX_TX_PKT_LEN  2044  ///<可发送的最大报文长度

#define rtol(r, n) ((r) << (n) | ((r) >> ((8 * sizeof(r)) - (n))))

///// 结构定义 ///////
struct rand_func {
	uint32_t x, y, z;
};

////////// 数据 /////////

static int glob_threads_num;
static int glob_loop_flag_main;
static int glob_loop_flag_thread;
static int glob_pkts_num;
static int glob_inc_len;
static int glob_print_pkt_flag;
static uint32_t glob_pkt_low_len;
static uint32_t glob_pkt_up_len;
static int glob_pkt_proto;
static int glob_ip_version;
static int glob_send_frame;
static uint32_t glob_sip;
static uint32_t glob_dip;
static struct in6_addr glob_ipv6_sip;
static struct in6_addr glob_ipv6_dip;

static struct rand_func randf[MAX_THREADS_NUM];

///计数器，每个线程分别计数
static ulong glob_pkts[MAX_THREADS_NUM][16];
static ulong glob_bytes[MAX_THREADS_NUM][16];

///////   函数列表  ///////////////

///@brief 输出程序帮助信息
///@param cmd 程序命令(输入)
void usage(char* cmd) 
{
    printf("Usage: %s ", cmd);
    printf("[-n pkt_num][-m thread_num][-l low_len][-u up_len]\n");
    printf("[-t pkt_type(proto)][-s src_ip][-d dst_ip][-v ip_version][-p] [-?]\n");
    printf("  -n pkts num to tx, default = continously\n");
    printf("  -m send thread number, default = %d\n", DEF_THREADS_NUM);
    printf("  -l min pkt lenth, default = %d\n", MIN_TX_PKT_LEN);
    printf("  -u max pkt lenth, default = %d\n", MAX_TX_PKT_LEN);
    printf("  -t pkt protocol and type, 17 = udp, 6 = tcp, default = random(TCP or UDP)\n");
    printf("  -s src ipv4 of pkt, default = random\n");
    printf("  -S src ipv6 of pkt, default = random\n");
    printf("  -d dst ipv4 of pkt, default = random\n");
    printf("  -D dst ipv6 of pkt, default = random\n");
    printf("  -v ip version, 4 = ipv4, 6 = ipv6, default = ipv4\n");
    printf("  -p print pkts\n");
    printf("  -? print usage message\n");
    printf("\n");
}


///@brief 打印发包信息的线程，每隔1秒显示一次
///@param ptr 线程建立时的指针，这里没有使用
void print_thread(void *ptr)
{
    ulong old_pkts[MAX_THREADS_NUM];
    ulong old_bytes[MAX_THREADS_NUM];
    int i;

    for (i = 0; i < MAX_THREADS_NUM; i++)
    {
        old_pkts[i] = 0;
        old_bytes[i] = 0;
    }
    
    while (glob_loop_flag_thread)
    {
        sleep(1); 
        printf("\nCurrent speed-----\n");
        for (i = 0; i < glob_threads_num; i++) 
        {
            printf("\t Thread[%d]:\t %8lu pps, %8lu M bps\n", i, 
                glob_pkts[i][0] - old_pkts[i], 
                (glob_bytes[i][0] - old_bytes[i]) >> 17);
            old_pkts[i] = glob_pkts[i][0];
            old_bytes[i] = glob_bytes[i][0];
        }
    }
}

///@brief 打印报文（检查收包内容）
///@param pkt 指向报文的指针
///@param len 报文长度
static void print_pkt(void *pkt, int len)
{
    int i;
    uint8_t *p8;
    static int pkt_print;
 
    ///找到报文附加头, 打印长度包括附加头
    pkt_print++;
    p8 = (uint8_t *)pkt;

    ///打印报文, 每16字节一行，128字节一段
    printf("\n====  pkt:%d  ====\n", pkt_print);
    for (i = 0; i < len; i++)
    {
        if ((i % 128 == 0))
        {
            printf("\n");
        }
        if ((i % 16 == 0))
        {
            printf("\n%03x: ", i / 16 + 1);
        }
        if (i % 8 == 0)
        printf(" ");
        printf("%02x ", p8[i]);
    }
    printf("\n");
}

///@brief 构造报文
///@param pkt 指向报文的指针
///@param len ip报文长度
///@param proto 报文类型，17为日志包（udp）,6为封堵包（tcp）
inline void build_ipv4_pkt(void *pkt, int len, int proto, uint32_t sip, uint32_t dip)
{
    struct iphdr *iph;
    
    ///找到报文ip头
    iph = (struct iphdr *)pkt;
    
    ///修改各字段
    iph->tot_len = htons(len);
    iph->protocol = proto;
    iph->saddr = sip;
    iph->daddr = dip;
    if (proto == 17)
    {
        *((uint16_t *)(pkt + 24)) = htons(len - 20);
    }
}

///@brief 构造ipv6报文
///@param pkt 指向报文的指针
///@param len ip报文长度
///@param proto 报文类型，17为日志包（udp）,6为封堵包（tcp）
inline void build_ipv6_pkt(void *pkt, int len, int proto, struct in6_addr sip, struct in6_addr dip)
{
    struct ip6_hdr *iph;
    
    ///找到报文ip头
    iph = (struct ip6_hdr *)pkt;
    
    ///修改各字段
    iph->ip6_plen = htons(len - sizeof(struct ip6_hdr));
    iph->ip6_nxt = proto;
    iph->ip6_src = sip;
    iph->ip6_dst = dip;
    if (proto == 17)
    {
        *((uint16_t *)(pkt + 44)) = htons(len - sizeof(struct ip6_hdr));
    }
}

static inline void my_srand(struct rand_func *f, uint32_t seed)
{
	unsigned n;

	f->x = 6247; 
	f->y = 3848; 
	f->z = 0;

	for (n = ((seed >> 22) & 0x3ff) + 20; n > 0; n--)
		f->x -= rtol(f->x, 21);
	for (n = ((seed >> 11) & 0x7ff) + 20; n > 0; n--)
		f->y = rtol(f->y, 11) - f->y;
	for (n = ((seed      ) & 0x7ff) + 20; n > 0; n--)
		f->z = 3286325185u - rtol(f->z, 19);
//	f->x = (seed      & 0x001fffffu) + 4027999010u;
//	f->y = ((seed>>7) & 0x0007ffffu) + 3993266363u;
//	f->z = (seed >> 13)              + 3605298456u;
}

static inline uint32_t my_rand(struct rand_func *f) 
{
	f->x -= rtol(f->x, 21);
	return f->x;
	/*
	f->x -= rtol(f->x, 21);
	f->y = rtol(f->y, 11) - f->y;
	f->z = 3286325185u - rtol(f->z, 19);

	return f->x ^ f->y ^ f->z;
	*/
	/*
	f->x = ~(2911329625u*f->x); 
	f->x = rtol(f->x, 17);

	f->y = 4031235431u * f->y; 
	f->y = rtol(f->y, 15);

	f->z = 3286325185u - rtol(f->z, 19);

	return (f->x + f->y) ^ f->z;
	*/
}

///@brief 发送报文的线程
///@param ptr 线程建立时的指针，这里指向线程的数据流号
void tx_thread_eth(void *ptr)
{
    int i, sid;
    uint32_t len, range, temp = 0;
    void *pkt;
    uint32_t sip, dip;
    struct in6_addr sip6, dip6;
    struct in6_addr zero_ip6 = IN6ADDR_ANY_INIT;
    uint8_t rand_sip6, rand_dip6;
    struct iphdr *iph;
    uint8_t p8[MAX_TX_PKT_LEN] = 
        {
            0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 
            0x99, 0xaa, 0xbb, 0xcc, 0x08, 0x00, //eth head
            0x45, 0x00, 0x00, 0x92, 0xc7, 0x6b, 0x00, 0x00, 
            0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
            0x02, 0x02, 0x02, 0x02, //ip head
            0x00, 0x50, 0x07, 0xbb, 0x78, 0x28, 0x73, 0x3e, 
            0x43, 0x23, 0x6a, 0x7c, 0x50, 0x12, 0x16, 0x58, 
            0x76, 0xd4, 0x00, 0x00//tcp head
        };
    uint8_t p8_v6[MAX_TX_PKT_LEN] = 
        {
            0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 
            0x99, 0xaa, 0xbb, 0xcc, 0x86, 0xdd, //eth head
            0x60, 0x00, 0x00, 0x00, 0x04, 0xd8, 0x06, 0x40,
            0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
            0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
            0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  
            0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  //IPV6报文基本头
            0xc9, 0x49, 0xe0, 0xf0, 0xce, 0xe0, 0xc4, 0xa0,
            0xe4, 0x76, 0x38, 0x0b, 0x50, 0x10, 0x40, 0x77,
            0x75, 0x2d, 0x00, 0x00  //TCP头
        };    
    
    ///获取缓冲区id
    sid = *((int *)ptr);

    my_srand(&randf[sid], time(NULL));

    rand_sip6 = memcmp(&glob_ipv6_sip, &zero_ip6, sizeof(zero_ip6));
    rand_dip6 = memcmp(&glob_ipv6_dip, &zero_ip6, sizeof(zero_ip6));

    
    ///把报文的应用数据设置为递增字节
    if (glob_ip_version == 4)
    {
        for (i = 0; i < MAX_TX_PKT_LEN - 54; i++)
        {
            p8[54 + i] = (uint8_t)i;
        }
    }
    else
    {
        for (i = 0; i < MAX_TX_PKT_LEN - 74; i++)
        {
            p8_v6[74 + i] = (uint8_t)i;
        }
    }
    
    ///循环，直到线程循环标志清零
    while (glob_loop_flag_thread)
    {
        ///--确定报文长度
//        temp = my_rand(&randf[sid]);
        range = glob_pkt_up_len - glob_pkt_low_len + 1;
        len = temp % range + glob_pkt_low_len;
	temp += glob_inc_len;
//	glob_pkt_proto = (temp & 0x1) ? IPPROTO_TCP : IPPROTO_UDP;

        ///--发送报文
        ///--如果没有缓冲区，则循环等待
        while (!(pkt = pag_getsendbuf_eth(sid, 0))) {
            usleep(5);
        }

        ///--把原始报文拷贝到发送缓冲区，根据参数对报文修改报文内容
        if (glob_ip_version == 4)
        {
//	    sip = (glob_sip == 0) ? my_rand(&randf[sid]) : glob_sip;
//	    dip = (glob_dip == 0) ? my_rand(&randf[sid]) : glob_dip;
	    sip = my_rand(&randf[sid]);
	    dip = my_rand(&randf[sid]);
            memcpy(pkt, p8, len);
//            build_ipv4_pkt(pkt + sizeof(struct ether_header), len - sizeof(struct ether_header), 
//                            glob_pkt_proto, sip, dip);
	        
	    ///找到报文ip头
	    pkt += sizeof(struct ether_header);
	    iph = (struct iphdr *)(pkt);

	    ///修改各字段
	    iph->tot_len = htons(len-sizeof(struct ether_header));
	    iph->protocol = glob_pkt_proto;
	    iph->saddr = sip;
	    iph->daddr = dip;
	    if (glob_pkt_proto == 17) {   
		    *((uint16_t *)(pkt + 24)) = htons(len - sizeof(struct ether_header) - 20);
	    }   
        }
        else
        {
		if (rand_sip6 == 0) {
			sip6.s6_addr32[0] = (uint32_t)my_rand(&randf[sid]);
			sip6.s6_addr32[1] = (uint32_t)my_rand(&randf[sid]);
			sip6.s6_addr32[2] = (uint32_t)my_rand(&randf[sid]);
			sip6.s6_addr32[3] = (uint32_t)my_rand(&randf[sid]);
		}
		if (rand_dip6 == 0) {
			dip6.s6_addr32[0] = (uint32_t)my_rand(&randf[sid]);
			dip6.s6_addr32[1] = (uint32_t)my_rand(&randf[sid]);
			dip6.s6_addr32[2] = (uint32_t)my_rand(&randf[sid]);
			dip6.s6_addr32[3] = (uint32_t)my_rand(&randf[sid]);
		}
            memcpy(pkt, p8_v6, len);
            build_ipv6_pkt(pkt + sizeof(struct ether_header), len - sizeof(struct ether_header), 
                            glob_pkt_proto, sip6, dip6);    
        }
        
        ///--发送报文
        if (pag_send_eth(sid, len, 0) == -1)
        {
            printf("pag_send_eth return -1\n");
            continue;
        }
       
        if (glob_print_pkt_flag)
        {
            print_pkt(pkt, len);
        }
        
        ///--增加报文计数
        glob_pkts[sid][0]++;
        glob_bytes[sid][0] += len;
    }
}

///@brief 发送报文的线程
///@param ptr 线程建立时的指针，这里指向线程的数据流号
void tx_thread(void *ptr)
{
    int i, sid;
    uint32_t len, range, temp = 0;
    void *pkt;
    uint32_t sip, dip;
    struct in6_addr sip6, dip6;
    struct in6_addr zero_ip6 = IN6ADDR_ANY_INIT;
    uint8_t rand_sip6, rand_dip6;
    struct iphdr *iph;
    uint8_t p8[MAX_TX_PKT_LEN-14] = 
        {
            0x45, 0x00, 0x00, 0x92, 0xc7, 0x6b, 0x00, 0x00, 
            0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
            0x02, 0x02, 0x02, 0x02, //ip head
            0x00, 0x50, 0x07, 0xbb, 0x78, 0x28, 0x73, 0x3e, 
            0x43, 0x23, 0x6a, 0x7c, 0x50, 0x12, 0x16, 0x58, 
            0x76, 0xd4, 0x00, 0x00//tcp head
        };
    uint8_t p8_v6[MAX_TX_PKT_LEN-14] = 
        {
            0x60, 0x00, 0x00, 0x00, 0x04, 0xd8, 0x06, 0x40,
            0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
            0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
            0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  
            0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  //IPV6报文基本头
            0xc9, 0x49, 0xe0, 0xf0, 0xce, 0xe0, 0xc4, 0xa0,
            0xe4, 0x76, 0x38, 0x0b, 0x50, 0x10, 0x40, 0x77,
            0x75, 0x2d, 0x00, 0x00  //TCP头
        };    
    
    ///获取缓冲区id
    sid = *((int *)ptr);

    my_srand(&randf[sid], time(NULL));

    rand_sip6 = memcmp(&glob_ipv6_sip, &zero_ip6, sizeof(zero_ip6));
    rand_dip6 = memcmp(&glob_ipv6_dip, &zero_ip6, sizeof(zero_ip6));

    
    ///把报文的应用数据设置为递增字节
    if (glob_ip_version == 4)
    {
        for (i = 0; i < MAX_TX_PKT_LEN - 54; i++)
        {
            p8[40 + i] = (uint8_t)i;
        }
    }
    else
    {
        for (i = 0; i < MAX_TX_PKT_LEN - 74; i++)
        {
            p8_v6[60 + i] = (uint8_t)i;
        }
    }
    
    ///循环，直到线程循环标志清零
    while (glob_loop_flag_thread)
    {
        ///--确定报文长度
        temp = my_rand(&randf[sid]);
        range = glob_pkt_up_len - glob_pkt_low_len + 1;
        len = temp % range + glob_pkt_low_len - sizeof(struct ether_header);
//	glob_pkt_proto = (temp & 0x1) ? IPPROTO_TCP : IPPROTO_UDP;

        ///--发送报文
        ///--如果没有缓冲区，则循环等待
        while (!(pkt = pag_getsendbuf(sid))) {
            usleep(5);
        }

        ///--把原始报文拷贝到发送缓冲区，根据参数对报文修改报文内容
        if (glob_ip_version == 4)
        {
	    sip = (glob_sip == 0) ? my_rand(&randf[sid]) : glob_sip;
	    dip = (glob_dip == 0) ? my_rand(&randf[sid]) : glob_dip;
            memcpy(pkt, p8, len);
//            build_ipv4_pkt(pkt, len - sizeof(struct ether_header), 
//                            glob_pkt_proto, sip, dip);
	        
	    ///找到报文ip头
//	    pkt += sizeof(struct ether_header);
	    iph = (struct iphdr *)(pkt);

	    ///修改各字段
	    iph->tot_len = htons(len);
	    iph->protocol = glob_pkt_proto;
	    iph->saddr = sip;
	    iph->daddr = dip;
	    if (glob_pkt_proto == 17) {   
		    *((uint16_t *)(pkt + 24)) = htons(len - 20);
	    }  
        }
        else
        {
		if (rand_sip6 == 0) {
			sip6.s6_addr32[0] = (uint32_t)my_rand(&randf[sid]);
			sip6.s6_addr32[1] = (uint32_t)my_rand(&randf[sid]);
			sip6.s6_addr32[2] = (uint32_t)my_rand(&randf[sid]);
			sip6.s6_addr32[3] = (uint32_t)my_rand(&randf[sid]);
		}
		if (rand_dip6 == 0) {
			dip6.s6_addr32[0] = (uint32_t)my_rand(&randf[sid]);
			dip6.s6_addr32[1] = (uint32_t)my_rand(&randf[sid]);
			dip6.s6_addr32[2] = (uint32_t)my_rand(&randf[sid]);
			dip6.s6_addr32[3] = (uint32_t)my_rand(&randf[sid]);
		}
            memcpy(pkt, p8_v6, len);
            build_ipv6_pkt(pkt, len, glob_pkt_proto, sip6, dip6);    
        }
        
        ///--发送报文
        if (pag_send((glob_pkt_proto == IPPROTO_TCP) ? 1 : 0, sid, len) == -1)
        {
            printf("pag_send_eth return -1\n");
            continue;
        }
       
        if (glob_print_pkt_flag)
        {
            print_pkt(pkt, len);
        }
        
        ///--增加报文计数
        glob_pkts[sid][0]++;
        glob_bytes[sid][0] += len;
    }
}

///@brief 信号响应函数，收到信号后把全局循环标志无效掉
void stop_loop(int sig)
{
    glob_loop_flag_main = 0;
}

void init_rand_func(void)  
{
	int i;

	for (i = 0; i < MAX_THREADS_NUM; i++) {
		randf[i].x = 0;
		randf[i].y = 0;
		randf[i].z = 0;
	}
}

///@brief 主函数
///@return 成功返回0，失败返回-1
int main(int argc, char *argv[])
{
    int c, i, ret;
    struct timeval ts, te;
    double sec; 
    ulong all_pkts, all_bytes;
    sigset_t set;
    pthread_t tx_tids[MAX_THREADS_NUM], print_tid;
    int sids[MAX_THREADS_NUM];
    
    ///打印使用信息
    usage(argv[0]);
    
    ///产生一个随机数种子
    init_rand_func();
//    srand(time(0)); 
    
    ///初始化参数
    glob_pkts_num = -1;
    glob_inc_len = 1;
    glob_pkt_low_len = MIN_TX_PKT_LEN;
    glob_pkt_up_len = MAX_TX_PKT_LEN;
    glob_pkt_proto = (rand() & 0x01) ? IPPROTO_TCP : IPPROTO_UDP;
    glob_sip = 0; //(uint32_t)rand();
    glob_dip = 0; //(uint32_t)rand();
    glob_threads_num = DEF_THREADS_NUM;
    glob_print_pkt_flag = 0;
    glob_send_frame = 0;
    glob_ip_version = 4;
    for (i = 0; i < 4; i++)
    {
        glob_ipv6_sip.s6_addr32[i] = 0; //(uint32_t)rand();
        glob_ipv6_dip.s6_addr32[i] = 0; //(uint32_t)rand();
    }
   
    while ((c = getopt(argc, argv, "n:m:l:u:i:t:s:S:d:D:v:pf?h")) != -1) 
    {
        switch (c)
        {
            case 'n':
                glob_pkts_num = atoi(optarg);
                break;
            case 'm':
                glob_threads_num = atoi(optarg);
                if (glob_threads_num > MAX_THREADS_NUM)
                {
                    printf("\nError: threads num invalid\n");
                    return -1;
                }
                break;
            case 'l':
                glob_pkt_low_len = atoi(optarg);
                if (glob_pkt_low_len < MIN_TX_PKT_LEN || glob_pkt_low_len > MAX_TX_PKT_LEN)
                {
                    printf("\nError: min pkt len invalid\n");
                    return -1;
                }
                break;
            case 'u':
                glob_pkt_up_len = atoi(optarg);
                if (glob_pkt_up_len < MIN_TX_PKT_LEN || glob_pkt_up_len > MAX_TX_PKT_LEN)
                {
                    printf("\nError: max pkt len invalid\n");
                    return -1;
                }
                break;
            case 'i':
                glob_inc_len = atoi(optarg);
                break;
            case 't':
                glob_pkt_proto = atoi(optarg);
                break;
            case 's':
                glob_sip = inet_addr(optarg);
                break;
            case 'S':
                ret = inet_pton(AF_INET6, optarg, &glob_ipv6_sip);
                if (ret <= 0)
                {
                    printf("\nError: get ipv6 src address failed\n");
                    return -1;   
                }
                break;
            case 'd':
                glob_dip = inet_addr(optarg);
                break;
            case 'D':
                ret = inet_pton(AF_INET6, optarg, &glob_ipv6_dip);
                if (ret <= 0)
                {
                    printf("\nError: get ipv6 dst address failed\n");
                    return -1;   
                }
                break;
	    case 'f':
		glob_send_frame = 1;
		break;
            case 'v':
                glob_ip_version = atoi(optarg);
                break;
            case 'p':
                glob_print_pkt_flag = 1;
                break;
            case '?':
            case 'h':
            default:
    //            usage(argv[0]);
                return 1;
        }
    }

    ///输入参数合法性判断
    if (glob_pkt_up_len < glob_pkt_low_len)
    {
        printf("Warning:send len rang[%d, %d]\n", glob_pkt_low_len, glob_pkt_up_len);
        usage(argv[0]);
        return -1;
    }
    
    if (glob_ip_version != 4 && glob_ip_version != 6)
    {
        usage(argv[0]);
        return -1;   
    }
    
    ///打开设备
    if (pag_open() < 0)
    {
        printf("### Error : can not open device! ###\n");
        return -1;
    }
    
    ///创建线程
    glob_loop_flag_thread = 1;
    for (i = 0; i < glob_threads_num; i++)
    {
	int ret;
	
        sids[i] = i;
	if (glob_send_frame == 1) 
        	ret = pthread_create(&tx_tids[i], NULL, (void *)tx_thread_eth, (void *)&sids[i]);  
	else
        	ret = pthread_create(&tx_tids[i], NULL, (void *)tx_thread, (void *)&sids[i]);  
        if (ret != 0)   
        {
            printf("### Error: Can not creat threads!\n");    
            pag_close();
            return -1;
        }
    }
    
    ///如果不打印报文内容，则打印报文速度
    if (!glob_print_pkt_flag)
    {
        if (pthread_create(&print_tid, NULL, (void *)print_thread, NULL) != 0)   
        {
            printf("### Error: Can not creat threads!\n");    
            pag_close();
            return -1;
        }
    }

    ///等待结束信号，或着发出期望的包数 
    sigemptyset(&set);
    sigprocmask(SIG_SETMASK, &set, NULL);
    signal(SIGINT, stop_loop);
    
    ///记录开始时间
    gettimeofday(&ts, NULL); 
    
    glob_loop_flag_main = 1;
    while (glob_loop_flag_main) 
    {
        usleep(10);  
        
        all_pkts = 0;
        all_bytes = 0;
        for (i = 0; i < glob_threads_num; i++) 
        {
            all_pkts += glob_pkts[i][0];
        }
        
        if (glob_pkts_num != 0 && glob_pkts_num <= all_pkts)
        {
            break;
        }
    }
    
    ///记录结束时间
    gettimeofday(&te, NULL);
    
    ///停止线程
    glob_loop_flag_thread = 0;
    for (i = 0; i < glob_threads_num; i++)
    {
        pthread_join(tx_tids[i], NULL);
    }
    if (!glob_print_pkt_flag)
    {
        pthread_join(print_tid, NULL);
    }
    
    ///关闭设备
    pag_close();   
    
    ///打印总结信息
    printf("\n");        
    printf("Pkts sent in threads: \n");
    
    sec = (float)(te.tv_sec - ts.tv_sec) + ((float)(te.tv_usec - ts.tv_usec)) / 1000000.f;
    all_pkts = 0;
    all_bytes = 0;
    for (i = 0; i < glob_threads_num; i++) 
    {
        printf("\t Thread[%d]:\t <%8lu> pkts, <%8lu M> bytes\n", i,
            glob_pkts[i][0], glob_bytes[i][0] >> 20);
        all_pkts += glob_pkts[i][0];
        all_bytes += glob_bytes[i][0];
    }
    printf("Total : %lu pkts,  %lu bytes\n", all_pkts, all_bytes);
    printf("Speed : < %u > pps, < %u M > bps\n", 
        (int)(all_pkts / sec), (int)((all_bytes >> 17) / sec));
    printf("\n");     
       
    return 0;
}