pktgen-dpdk使用dpdk加速包的发送接收,也可以发送接收pcap包,命令行如下:
./app/app/x86_64-native-linuxapp-gcc/pktgen -l 0-4 -n 3 -- -P -m "[11:3].0,[2:4].1" -s 0:[.pcap_filepath] (pktgen-dpdk.3.4.8)
下面的源码讲述采用的也是3.4.8版本,主要讲述源码中的设计逻辑。
1. pktgen-dpdk逻辑设计
在pktgen-main.c文件中包含了main主入口函数main()以及参数配置函数pktgen_parse_args(),其中pktgen结构体是所有的参数都是用的,参数配置函数主要是对pktgen结构中个成员赋值。
main函数中pktgen初始化如下:
memset(&pktgen, 0, sizeof(pktgen));
pktgen.flags = PRINT_LABELS_FLAG;
pktgen.ident = 0x1234;
pktgen.nb_rxd = DEFAULT_RX_DESC;
pktgen.nb_txd = DEFAULT_TX_DESC;
pktgen.nb_ports_per_page = DEFAULT_PORTS_PER_PAGE;
if ( (pktgen.l2p = l2p_create()) == NULL)
pktgen_log_panic("Unable to create l2p");
pktgen.portdesc_cnt = get_portdesc(pktgen.portlist,
pktgen.portdesc,
RTE_MAX_ETHPORTS,
0);
然后初始化log,cpu
pktgen_init_log();
pktgen_cpu_init();
配置初始化port信息
void
pktgen_config_ports(void)
{
uint32_t lid, pid, i, s, q, sid;
rxtx_t rt;
pkt_seq_t *pkt;
port_info_t *info;
char buff[RTE_MEMZONE_NAMESIZE];
int32_t ret, cache_size;
char output_buff[256] = { 0 };
uint64_t ticks;
/* Find out the total number of ports in the system. */
/* We have already blacklisted the ones we needed to in main routine. */
pktgen.nb_ports = rte_eth_dev_count();
if (pktgen.nb_ports > RTE_MAX_ETHPORTS)
pktgen.nb_ports = RTE_MAX_ETHPORTS;
if (pktgen.nb_ports == 0)
pktgen_log_panic("*** Did not find any ports to use ***");
pktgen.starting_port = 0;
/* Setup the number of ports to display at a time */
if (pktgen.nb_ports > pktgen.nb_ports_per_page)
pktgen.ending_port = pktgen.starting_port +
pktgen.nb_ports_per_page;
else
pktgen.ending_port = pktgen.starting_port + pktgen.nb_ports;
pg_port_matrix_dump(pktgen.l2p);
....
....
for (s = 0; s < NUM_TOTAL_PKTS; s++)
pktgen_port_defaults(pid, s);
.....
其中主要获取cpu端口数,以及填充结构体info,其中函数pktgen_port)defaults()主要设置一些默认显示的信息,其中里面的函数pktgen_packet_rate()计算传输率,此函数作用与动态的展示发送接收的数据参数。
main()往下调用的函数是pktgen_clear_display(),
void
pktgen_clear_display(void)
{
if (!scrn_is_paused()) {
scrn_pause();
scrn_cls();
scrn_pos(100, 1);
pktgen_update_display();
scrn_resume();
pktgen_page_display(NULL, NULL);
}
}
其中pktgen_update_dsiplay()修改pktgen.flag标志
void
pktgen_page_display(struct rte_timer *tim __rte_unused, void *arg __rte_unused)
{
static unsigned int update_display = 1;
/* Leave if the screen is paused */
if (scrn_is_paused())
return;
scrn_save();
if (pktgen.flags & UPDATE_DISPLAY_FLAG) {
pktgen.flags &= ~UPDATE_DISPLAY_FLAG;
update_display = 1;
}
update_display--;
if (update_display == 0) {
update_display = UPDATE_DISPLAY_TICK_INTERVAL;
_page_display();
if (pktgen.flags & PRINT_LABELS_FLAG)
pktgen.flags &= ~PRINT_LABELS_FLAG;
}
scrn_restore();
pktgen_print_packet_dump();
}
函数中主要执行两个函数_page_display(),pktgen_printf_paket_dump(),前者调用pktgen_page_stats(),显示端口上的统计信息,
void
pktgen_page_stats(void)
{
port_info_t *info;
unsigned int pid, col, row;
struct rte_eth_stats *rate, *cumm, *prev;
unsigned sp;
char buff[32];
int display_cnt;
if (pktgen.flags & PRINT_LABELS_FLAG)
pktgen_print_static_data();
...
...
其中函数pktgen_print_static_data()显示一些静态数据,动态数据主要通过info结构体传输。
然后main函数调用rte_timer_setup()
void
rte_timer_setup(void)
{
int lcore_id = rte_get_master_lcore();
/* init RTE timer library */
rte_timer_subsystem_init();
/* init timer structures */
rte_timer_init(&timer0);
rte_timer_init(&timer1);
/* load timer0, every 1/2 seconds, on Display lcore, reloaded automatically */
rte_timer_reset(&timer0,
UPDATE_DISPLAY_TICK_RATE,
PERIODICAL,
lcore_id,
pktgen_page_display,
NULL);
/* load timer1, every second, on timer lcore, reloaded automatically */
rte_timer_reset(&timer1,
pktgen.hz,
PERIODICAL,
lcore_id,
pktgen_process_stats,
NULL);
}
其中timer0,用于更新显示,timer1用于统计数据。
main函数接着调用pktgen_cli_start(),最终调用cli_start(),用于执行运行时参数。
2. pktgen 流量生成器
下面说一下发送接收包,在main函数的前部分还有一个重要的函数:
ret = rte_eal_remote_launch(pktgen_launch_one_lcore, NULL, i);
/* Configure and initialize the ports */
pktgen_config_ports();
pktgen_log_info("");
pktgen_log_info("=== Display processing on lcore %d", rte_lcore_id());
/* launch per-lcore init on every lcore except master and master + 1 lcores */
for (i = 0; i < RTE_MAX_LCORE; i++) {
if ( (i == rte_get_master_lcore()) || !rte_lcore_is_enabled(i) )
continue;
ret = rte_eal_remote_launch(pktgen_launch_one_lcore, NULL, i);
if (ret != 0)
pktgen_log_error("Failed to start lcore %d, return %d", i, ret);
}
rte_delay_ms(1000); /* Wait for the lcores to start up. */
其中pktgen_launch-one_lcore()调用pktgen_main_tx_loop(),pktgen_mian_rx_loop(),pktgen_main_rxtx_loop()。
这些函数相应的调用函数,同时更新pktgen结构或是其中的成员,以pktgen_main_rxtx_loop()为例。
pktgen_main_rxtx_loop() -> pktgen_main_transmit() -> pktgen_send_pkts() ->pktgen_send_burst() -> _send_burst_..()
-> pkt_do_tx_tap() -> write()
下面说一下pktgen作为流量生成器的一些个人理解。
pktgen作为linux内核模块的一部分,因此可以直接加载内核模块generate流量(lsmod pktgen),网上参考资料说是产生udp类型的packet,在此详细探讨一下。
在pktgen_main_transmit()中
static __inline__ void
pktgen_main_transmit(port_info_t *info, uint16_t qid)
{
struct rte_mempool *mp = NULL;
uint32_t flags;
flags = rte_atomic32_read(&info->port_flags);
/*
* Transmit ARP/Ping packets if needed
*/
if ((flags & SEND_ARP_PING_REQUESTS))
pktgen_send_special(info, flags);
/* When not transmitting on this port then continue. */
if (flags & SENDING_PACKETS) {
mp = info->q[qid].tx_mp;
if (flags & (SEND_RANGE_PKTS | SEND_PCAP_PKTS | SEND_SEQ_PKTS)) {
if (flags & SEND_RANGE_PKTS)
mp = info->q[qid].range_mp;
else if (flags & SEND_SEQ_PKTS)
mp = info->q[qid].seq_mp;
else if (flags & SEND_PCAP_PKTS)
mp = info->q[qid].pcap_mp;
}
if (rte_atomic32_read(&info->q[qid].flags) & CLEAR_FAST_ALLOC_FLAG)
pktgen_setup_packets(info, mp, qid);
pktgen_send_pkts(info, qid, mp);
}
flags = rte_atomic32_read(&info->q[qid].flags);
if (flags & DO_TX_FLUSH)
pktgen_tx_flush(info, qid);
}
最终调用函数pktgen_setup_cb(),调用了pktgen_range_ctor()和pktgen_packet_ctor()
其中结构体:
typedef struct pkt_seq_s {
/* Packet type and information */
struct ether_addr eth_dst_addr; /**< Destination Ethernet address */
struct ether_addr eth_src_addr; /**< Source Ethernet address */
struct cmdline_ipaddr ip_src_addr; /**< Source IPv4 address also used for IPv6 */
struct cmdline_ipaddr ip_dst_addr; /**< Destination IPv4 address */
uint32_t ip_mask; /**< IPv4 Netmask value */
uint16_t sport; /**< Source port value */
uint16_t dport; /**< Destination port value */
uint16_t ethType; /**< IPv4 or IPv6 */
uint16_t ipProto; /**< TCP or UDP or ICMP */
uint16_t vlanid; /**< VLAN ID value if used */
uint8_t cos; /**< 802.1p cos value if used */
uint8_t tos; /**< tos value if used */
uint16_t ether_hdr_size;/**< Size of Ethernet header in packet for VLAN ID */
uint32_t mpls_entry; /**< MPLS entry if used */
uint16_t qinq_outerid; /**< Outer VLAN ID if Q-in-Q */
uint16_t qinq_innerid; /**< Inner VLAN ID if Q-in-Q */
uint32_t gre_key; /**< GRE key if used */
uint16_t pktSize; /**< Size of packet in bytes not counting FCS */
uint16_t pad0;
uint32_t gtpu_teid; /**< GTP-U TEID, if UDP dport=2152 */
uint8_t seq_enabled; /**< Enable or disable this sequence through GUI */
pkt_hdr_t hdr __rte_cache_aligned; /**< Packet header data */
uint8_t pad[MBUF_SIZE - sizeof(pkt_hdr_t)];
} pkt_seq_t __rte_cache_aligned;
void pktgen_packet_ctor(port_info_t *info, int32_t seq_idx, int32_t type){
.....
if (likely(pkt->ethType == ETHER_TYPE_IPv4)) {
if (likely(pkt->ipProto == PG_IPPROTO_TCP)) { //构造TCP
if (pkt->dport != PG_IPPROTO_L4_GTPU_PORT) {
/* Construct the TCP header */
pktgen_tcp_hdr_ctor(pkt, l3_hdr, ETHER_TYPE_IPv4);
/* IPv4 Header constructor */
pktgen_ipv4_ctor(pkt, l3_hdr);
} else {
/* Construct the GTP-U header */
pktgen_gtpu_hdr_ctor(pkt, l3_hdr, pkt->ipProto,
GTPu_VERSION | GTPu_PT_FLAG, 0, 0, 0);
/* Construct the TCP header */
pktgen_tcp_hdr_ctor(pkt, l3_hdr, ETHER_TYPE_IPv4);
/* IPv4 Header constructor */
pktgen_ipv4_ctor(pkt, l3_hdr);
}
} else if (pkt->ipProto == PG_IPPROTO_UDP) {
if (pkt->dport != PG_IPPROTO_L4_GTPU_PORT) {
/* Construct the UDP header */
pktgen_udp_hdr_ctor(pkt, l3_hdr, ETHER_TYPE_IPv4);
/* IPv4 Header constructor */
pktgen_ipv4_ctor(pkt, l3_hdr);
} else {
/* Construct the GTP-U header */
pktgen_gtpu_hdr_ctor(pkt, l3_hdr, pkt->ipProto,
GTPu_VERSION | GTPu_PT_FLAG, 0, 0, 0);
/* Construct the UDP header */
pktgen_udp_hdr_ctor(pkt, l3_hdr, ETHER_TYPE_IPv4);
/* IPv4 Header constructor */
pktgen_ipv4_ctor(pkt, l3_hdr);
}
} else if (pkt->ipProto == PG_IPPROTO_ICMP) {
udpip_t *uip;
icmpv4Hdr_t *icmp;
/* Start from Ethernet header */
uip = (udpip_t *)l3_hdr;
/* Create the ICMP header */
uip->ip.src = htonl(pkt->ip_src_addr.addr.ipv4.s_addr);
uip->ip.dst = htonl(pkt->ip_dst_addr.addr.ipv4.s_addr);
tlen = pkt->pktSize - (pkt->ether_hdr_size + sizeof(ipHdr_t));
uip->ip.len = htons(tlen);
uip->ip.proto = pkt->ipProto;
icmp = (icmpv4Hdr_t *)&uip->udp;
icmp->code = 0;
if ( (type == -1) || (type == ICMP4_TIMESTAMP)) {
icmp->type =
ICMP4_TIMESTAMP;
icmp->data.timestamp.ident = 0x1234;
icmp->data.timestamp.seq = 0x5678;
icmp->data.timestamp.originate = 0x80004321;
icmp->data.timestamp.receive = 0;
icmp->data.timestamp.transmit = 0;
} else if (type == ICMP4_ECHO) {
icmp->type = ICMP4_ECHO;
icmp->data.echo.ident = 0x1234;
icmp->data.echo.seq = 0x5678;
icmp->data.echo.data = 0;
}
icmp->cksum = 0;
/* ICMP4_TIMESTAMP_SIZE */
tlen = pkt->pktSize - (pkt->ether_hdr_size + sizeof(ipHdr_t));
icmp->cksum = cksum(icmp, tlen, 0);
if (icmp->cksum == 0)
icmp->cksum = 0xFFFF;
/* IPv4 Header constructor */
pktgen_ipv4_ctor(pkt, l3_hdr);
}
} else if (pkt->ethType == ETHER_TYPE_IPv6) {
if (pkt->ipProto == PG_IPPROTO_TCP) {
/* Construct the TCP header */
pktgen_tcp_hdr_ctor(pkt, l3_hdr, ETHER_TYPE_IPv6);
/* IPv6 Header constructor */
pktgen_ipv6_ctor(pkt, l3_hdr);
} else if (pkt->ipProto == PG_IPPROTO_UDP) {
/* Construct the UDP header */
pktgen_udp_hdr_ctor(pkt, l3_hdr, ETHER_TYPE_IPv6);
/* IPv6 Header constructor */
pktgen_ipv6_ctor(pkt, l3_hdr);
}
} else if (pkt->ethType == ETHER_TYPE_ARP) {
/* Start from Ethernet header */
arpPkt_t *arp = (arpPkt_t *)l3_hdr;
arp->hrd = htons(1);
arp->pro = htons(ETHER_TYPE_IPv4);
arp->hln = ETHER_ADDR_LEN;
arp->pln = 4;
/* FIXME make request/reply operation selectable by user */
arp->op = htons(2);
ether_addr_copy(&pkt->eth_src_addr,
(struct ether_addr *)&arp->sha);
arp->spa._32 = htonl(pkt->ip_src_addr.addr.ipv4.s_addr);
ether_addr_copy(&pkt->eth_dst_addr,
(struct ether_addr *)&arp->tha);
arp->tpa._32 = htonl(pkt->ip_dst_addr.addr.ipv4.s_addr);
}
这里主要是根据类型填充首部结构,其中pktgen_tcp_hdr_ctor()填充ip首部和tcp首部,return l3_hdr,然后在调用pktgen_ipv4_ctor()填充ip header.
对于pktgen_range_ctor(), 主要是根据range_info_t结构体重的最大最下值以及步长递增地修改首部结构。
从上面可以看书pktgen不只是产生udp类型的数据包,也可以产生其他类型的数据包,但是只是产生了流量,而没有payload。
3. pktgen-dpdk发送pcap包
发送pcap数据包需要使用 -s:[pcap.filepcath]参数。
static int pktgen_parse_args(int argc, char **argv)
{
....
case 's': /* Read a PCAP packet capture file (stream) */
port = strtol(optarg, NULL, 10); //将字符串port转换为长整型port
p = strchr(optarg, ':'); // ++p指向待发送的pcap文件
if ( (p == NULL) ||
(pktgen.info[port].pcap =
_pcap_open(++p, port)) == NULL) {
pktgen_log_error(
"Invalid PCAP filename (%s) must include port number as P:filename",
optarg);
pktgen_usage(prgname);
return -1;
.......
}
此处p指向pcap文件(此处为绝对路劲)
pcap_info_t *
_pcap_open(char *filename, uint16_t port)
{
pcap_info_t *pcap = NULL;
if (filename == NULL) {
printf("%s: filename is NULL\n", __FUNCTION__);
goto leave;
}
pcap = (pcap_info_t *)rte_malloc("PCAP info",
sizeof(pcap_info_t),
RTE_CACHE_LINE_SIZE);
if (pcap == NULL) {
printf("%s: malloc failed for pcap_info_t structure\n",
__FUNCTION__);
goto leave;
}
memset((char *)pcap, 0, sizeof(pcap_info_t));
pcap->fd = fopen((const char *)filename, "r");
if (pcap->fd == NULL) {
printf("%s: failed for (%s)\n", __FUNCTION__, filename);
goto leave;
}
if (fread(&pcap->info, 1, sizeof(pcap_hdr_t),
pcap->fd) != sizeof(pcap_hdr_t) ) {
printf("%s: failed to read the file header\n", __FUNCTION__);
goto leave;
}
/* Default to little endian format. */
pcap->endian = LITTLE_ENDIAN;
pcap->filename = strdup(filename);
/* Make sure we have a valid PCAP file for Big or Little Endian formats. */
if ( (pcap->info.magic_number != PCAP_MAGIC_NUMBER) &&
(pcap->info.magic_number != ntohl(PCAP_MAGIC_NUMBER)) ) {
printf("%s: Magic Number does not match!\n", __FUNCTION__);
fflush(stdout);
goto leave;
}
/* Convert from big-endian to little-endian. */
if (pcap->info.magic_number == ntohl(PCAP_MAGIC_NUMBER) ) {
printf(
"PCAP: Big Endian file format found, converting to little endian\n");
pcap->endian = BIG_ENDIAN;
pcap->info.magic_number = ntohl(pcap->info.magic_number);
pcap->info.network = ntohl(pcap->info.network);
pcap->info.sigfigs = ntohl(pcap->info.sigfigs);
pcap->info.snaplen = ntohl(pcap->info.snaplen);
pcap->info.thiszone = ntohl(pcap->info.thiszone);
pcap->info.version_major = ntohs(pcap->info.version_major);
pcap->info.version_minor = ntohs(pcap->info.version_minor);
}
_pcap_info(pcap, port, 0);
return pcap;
leave:
_pcap_close(pcap);
fflush(stdout);
return NULL;
}
在 pkgen_config_ports()中
/* Setup the PCAP file for each port */
if (pktgen.info[pid].pcap != NULL)
if (pktgen_pcap_parse(pktgen.info[pid].pcap, info, q) == -1)
pktgen_log_panic("Cannot load PCAP file for port %d", pid);
/* Find out the link speed to program the WTHRESH value correctly. */
pktgen_get_link_status(info, pid, 0);
其中调用了pktgen_pcap_parse(),在里面读取pcap文件数据的函数是_pcap_read()
size_t
_pcap_read(pcap_info_t *pcap,
pcaprec_hdr_t *pHdr,
char *pktBuff,
uint32_t bufLen)
{
do {
if (fread(pHdr, 1, sizeof(pcaprec_hdr_t),
pcap->fd) != sizeof(pcaprec_hdr_t) )
return 0;
/* Convert the packet header to the correct format. */
_pcap_convert(pcap, pHdr);
/* Skip packets larger then the buffer size. */
if (pHdr->incl_len > bufLen) {
(void)fseek(pcap->fd, pHdr->incl_len, SEEK_CUR);
return pHdr->incl_len;
}
return fread(pktBuff, 1, pHdr->incl_len, pcap->fd);
} while (1);
}
然后调用dpdk的rte_mempool_create()
可以看到每次只能发送一个pcap文件,如果需要发送多个pcap文件,则需要解析多个pcap文件路劲,赋值给pcap.fd(可以使用缓存预先存储)。