325 struct targ {  struct
505 	bcopy(targ->src_mac, eh->ether_shost, 6);  in extract_mac_range()
506 	p = index(targ->g->src_mac, '-');  in extract_mac_range()
508 		targ->src_mac_range = atoi(p+1);  in extract_mac_range()
510 	bcopy(ether_aton(targ->g->dst_mac), targ->dst_mac, 6);  in extract_mac_range()
511 	bcopy(targ->dst_mac, eh->ether_dhost, 6);  in extract_mac_range()
512 	p = index(targ->g->dst_mac, '-');  in extract_mac_range()
514 		targ->dst_mac_range = atoi(p+1);  in extract_mac_range()
563 static struct targ *targs;
829 update_ip(struct pkt *pkt, struct targ *t)  in update_ip()
915 update_ip6(struct pkt *pkt, struct targ *t)  in update_ip6()
1000 update_addresses(struct pkt *pkt, struct targ *t)  in update_addresses()
1077 update_size(struct pkt *pkt, struct targ *t, int size)  in update_size()
1092 initialize_packet(struct targ *targ)  in initialize_packet()  argument
1094 	struct pkt *pkt = &targ->pkt;  in initialize_packet()
1102 	const char *payload = targ->g->options & OPT_INDIRECT ?  in initialize_packet()
1113 	if (targ->g->packet_file != NULL) {  in initialize_packet()
1114 		if ((file = pcap_open_offline(targ->g->packet_file,  in initialize_packet()
1117 			    targ->g->packet_file);  in initialize_packet()
1120 			    targ->g->packet_file);  in initialize_packet()
1121 		if ((targ->frame = malloc(header->caplen)) == NULL)  in initialize_packet()
1123 		bcopy(packet, (unsigned char *)targ->frame, header->caplen);  in initialize_packet()
1124 		targ->g->pkt_size = header->caplen;  in initialize_packet()
1130 	paylen = targ->g->pkt_size - sizeof(*eh) -  in initialize_packet()
1131 	    (targ->g->af == AF_INET ? sizeof(ip): sizeof(ip6));  in initialize_packet()
1137 		bcopy(payload, PKT(pkt, body, targ->g->af) + i, l0);  in initialize_packet()
1139 	PKT(pkt, body, targ->g->af)[i - 1] = '\0';  in initialize_packet()
1143 	bcopy(&targ->g->src_mac.start, eh->ether_shost, 6);  in initialize_packet()
1144 	bcopy(&targ->g->dst_mac.start, eh->ether_dhost, 6);  in initialize_packet()
1146 	if (targ->g->af == AF_INET) {  in initialize_packet()
1154 		ip.ip_len = htons(targ->g->pkt_size - sizeof(*eh));  in initialize_packet()
1159 		ip.ip_dst.s_addr = htonl(targ->g->dst_ip.ipv4.start);  in initialize_packet()
1160 		ip.ip_src.s_addr = htonl(targ->g->src_ip.ipv4.start);  in initialize_packet()
1172 		ip6.ip6_src = targ->g->src_ip.ipv6.start;  in initialize_packet()
1173 		ip6.ip6_dst = targ->g->dst_ip.ipv6.start;  in initialize_packet()
1177 	udp.uh_sport = htons(targ->g->src_ip.port0);  in initialize_packet()
1178 	udp.uh_dport = htons(targ->g->dst_ip.port0);  in initialize_packet()
1180 	if (targ->g->af == AF_INET) {  in initialize_packet()
1261 		int size, struct targ *t, u_int count, int options)  in send_packets()
1385 	struct targ *targ = (struct targ *) data;  in ping_body()  local
1386 	struct pollfd pfd = { .fd = targ->fd, .events = POLLIN };  in ping_body()
1387 	struct netmap_if *nifp = targ->nmd->nifp;  in ping_body()
1393 	uint64_t sent = 0, n = targ->g->npackets;  in ping_body()
1397 	int rate_limit = targ->g->tx_rate, tosend = 0;  in ping_body()
1399 	frame = (char*)&targ->pkt + sizeof(targ->pkt.vh) - targ->g->virt_header;  in ping_body()
1400 	size = targ->g->pkt_size + targ->g->virt_header;  in ping_body()
1403 	if (targ->g->nthreads > 1) {  in ping_body()
1408 	if (targ->g->af == AF_INET6) {  in ping_body()
1416 		targ->tic = timespec_add(now, (struct timespec){2,0});  in ping_body()
1417 		targ->tic.tv_nsec = 0;  in ping_body()
1418 		wait_time(targ->tic);  in ping_body()
1419 		nexttime = targ->tic;  in ping_body()
1421 	while (!targ->cancel && (n == 0 || sent < n)) {  in ping_body()
1422 		struct netmap_ring *ring = NETMAP_TXRING(nifp, targ->nmd->first_tx_ring);  in ping_body()
1429 			tosend = targ->g->burst;  in ping_body()
1430 			nexttime = timespec_add(nexttime, targ->g->tx_period);  in ping_body()
1434 		limit = rate_limit ? tosend : targ->g->burst;  in ping_body()
1459 		targ->ctr.pkts = sent;  in ping_body()
1460 		targ->ctr.bytes = sent*size;  in ping_body()
1461 		targ->ctr.events = event;  in ping_body()
1467 			D("TXSYNC error on queue %d: %s", targ->me,  in ping_body()
1475 			D("poll error on queue %d: %s", targ->me,  in ping_body()
1484 		for (i = targ->nmd->first_rx_ring;  in ping_body()
1485 			i <= targ->nmd->last_rx_ring; i++) {  in ping_body()
1559 		if (rx < m && ts.tv_sec <= 3 && !targ->cancel)  in ping_body()
1569 	targ->completed = 1;  in ping_body()
1572 	targ->used = 0;  in ping_body()
1584 	struct targ *targ = (struct targ *) data;  in pong_body()  local
1585 	struct pollfd pfd = { .fd = targ->fd, .events = POLLIN };  in pong_body()
1586 	struct netmap_if *nifp = targ->nmd->nifp;  in pong_body()
1589 	uint64_t sent = 0, n = targ->g->npackets;  in pong_body()
1591 	if (targ->g->nthreads > 1) {  in pong_body()
1599 	if (targ->g->af == AF_INET6) {  in pong_body()
1603 	while (!targ->cancel && (n == 0 || sent < n)) {  in pong_body()
1611 			D("poll error on queue %d: %s", targ->me,  in pong_body()
1616 		txring = NETMAP_TXRING(nifp, targ->nmd->first_tx_ring);  in pong_body()
1620 		for (i = targ->nmd->first_rx_ring; i <= targ->nmd->last_rx_ring; i++) {  in pong_body()
1660 		targ->ctr.pkts = sent;  in pong_body()
1666 	targ->completed = 1;  in pong_body()
1669 	targ->used = 0;  in pong_body()
1678 	struct targ *targ = (struct targ *) data;  in sender_body()  local
1679 	struct pollfd pfd = { .fd = targ->fd, .events = POLLOUT };  in sender_body()
1683 	uint64_t n = targ->g->npackets / targ->g->nthreads;  in sender_body()
1686 	int options = targ->g->options;  in sender_body()
1688 	int rate_limit = targ->g->tx_rate;  in sender_body()
1689 	struct pkt *pkt = &targ->pkt;  in sender_body()
1693 	if (targ->frame == NULL) {  in sender_body()
1694 		frame = (char *)pkt + sizeof(pkt->vh) - targ->g->virt_header;  in sender_body()
1695 		size = targ->g->pkt_size + targ->g->virt_header;  in sender_body()
1697 		frame = targ->frame;  in sender_body()
1698 		size = targ->g->pkt_size;  in sender_body()
1701 	D("start, fd %d main_fd %d", targ->fd, targ->g->main_fd);  in sender_body()
1702 	if (setaffinity(targ->thread, targ->affinity))  in sender_body()
1706 	clock_gettime(CLOCK_REALTIME_PRECISE, &targ->tic);  in sender_body()
1708 		targ->tic = timespec_add(targ->tic, (struct timespec){2,0});  in sender_body()
1709 		targ->tic.tv_nsec = 0;  in sender_body()
1710 		wait_time(targ->tic);  in sender_body()
1711 		nexttime = targ->tic;  in sender_body()
1713 	if (targ->g->dev_type == DEV_TAP) {  in sender_body()
1714 	    D("writing to file desc %d", targ->g->main_fd);  in sender_body()
1716 	    for (i = 0; !targ->cancel && (n == 0 || sent < n); i++) {  in sender_body()
1717 		if (write(targ->g->main_fd, frame, size) != -1)  in sender_body()
1719 		update_addresses(pkt, targ);  in sender_body()
1721 			targ->ctr.pkts = sent;  in sender_body()
1722 			targ->ctr.bytes = sent*size;  in sender_body()
1723 			targ->ctr.events = sent;  in sender_body()
1728     } else if (targ->g->dev_type == DEV_PCAP) {  in sender_body()
1729 	    pcap_t *p = targ->g->p;  in sender_body()
1731 	    for (i = 0; !targ->cancel && (n == 0 || sent < n); i++) {  in sender_body()
1734 		update_addresses(pkt, targ);  in sender_body()
1736 			targ->ctr.pkts = sent;  in sender_body()
1737 			targ->ctr.bytes = sent*size;  in sender_body()
1738 			targ->ctr.events = sent;  in sender_body()
1745 	u_int bufsz, frag_size = targ->g->frag_size;  in sender_body()
1747 	nifp = targ->nmd->nifp;  in sender_body()
1748 	txring = NETMAP_TXRING(nifp, targ->nmd->first_tx_ring);  in sender_body()
1752 	targ->frag_size = targ->g->pkt_size / targ->frags;  in sender_body()
1753 	if (targ->frag_size > frag_size) {  in sender_body()
1754 		targ->frags = targ->g->pkt_size / frag_size;  in sender_body()
1755 		targ->frag_size = frag_size;  in sender_body()
1756 		if (targ->g->pkt_size % frag_size != 0)  in sender_body()
1757 			targ->frags++;  in sender_body()
1759 	D("frags %u frag_size %u", targ->frags, targ->frag_size);  in sender_body()
1762 	for (i = targ->nmd->first_tx_ring; i <= targ->nmd->last_tx_ring; i++) {  in sender_body()
1773 	while (!targ->cancel && (n == 0 || sent < n)) {  in sender_body()
1777 			tosend = targ->g->burst;  in sender_body()
1778 			nexttime = timespec_add(nexttime, targ->g->tx_period);  in sender_body()
1788 			D("ioctl error on queue %d: %s", targ->me,  in sender_body()
1794 			if (targ->cancel)  in sender_body()
1796 			D("poll error on queue %d: %s", targ->me,  in sender_body()
1802 				targ->nmd->first_tx_ring, targ->nmd->last_tx_ring);  in sender_body()
1809 		for (i = targ->nmd->first_tx_ring; i <= targ->nmd->last_tx_ring; i++) {  in sender_body()
1811 			uint64_t limit = rate_limit ?  tosend : targ->g->burst;  in sender_body()
1822 			if (targ->g->pkt_min_size > 0) {  in sender_body()
1823 				size = nrand48(targ->seed) %  in sender_body()
1824 					(targ->g->pkt_size - targ->g->pkt_min_size) +  in sender_body()
1825 					targ->g->pkt_min_size;  in sender_body()
1826 				update_size(pkt, targ, size);  in sender_body()
1828 			m = send_packets(txring, pkt, frame, size, targ,  in sender_body()
1835 			targ->ctr.pkts = sent;  in sender_body()
1836 			targ->ctr.bytes += m*size;  in sender_body()
1837 			targ->ctr.events = event;  in sender_body()
1854 	for (i = targ->nmd->first_tx_ring; i <= targ->nmd->last_tx_ring; i++) {  in sender_body()
1856 		while (!targ->cancel && nm_tx_pending(txring)) {  in sender_body()
1865 	clock_gettime(CLOCK_REALTIME_PRECISE, &targ->toc);  in sender_body()
1866 	targ->completed = 1;  in sender_body()
1867 	targ->ctr.pkts = sent;  in sender_body()
1868 	targ->ctr.bytes = sent*size;  in sender_body()
1869 	targ->ctr.events = event;  in sender_body()
1872 	targ->used = 0;  in sender_body()
1925 	struct targ *targ = (struct targ *) data;  in receiver_body()  local
1926 	struct pollfd pfd = { .fd = targ->fd, .events = POLLIN };  in receiver_body()
1931 	uint64_t n = targ->g->npackets / targ->g->nthreads;  in receiver_body()
1935 	if (setaffinity(targ->thread, targ->affinity))  in receiver_body()
1939 		targ->g->ifname, targ->fd, targ->g->main_fd);  in receiver_body()
1941 	for (;!targ->cancel;) {  in receiver_body()
1957 	clock_gettime(CLOCK_REALTIME_PRECISE, &targ->tic);  in receiver_body()
1958     if (targ->g->dev_type == DEV_TAP) {  in receiver_body()
1959 	while (!targ->cancel && (n == 0 || targ->ctr.pkts < n)) {  in receiver_body()
1962 		i = read(targ->g->main_fd, buf, sizeof(buf));  in receiver_body()
1964 			targ->ctr.pkts++;  in receiver_body()
1965 			targ->ctr.bytes += i;  in receiver_body()
1966 			targ->ctr.events++;  in receiver_body()
1970     } else if (targ->g->dev_type == DEV_PCAP) {  in receiver_body()
1971 	while (!targ->cancel && (n == 0 || targ->ctr.pkts < n)) {  in receiver_body()
1973 		pcap_dispatch(targ->g->p, targ->g->burst, receive_pcap,  in receiver_body()
1974 			(u_char *)&targ->ctr);  in receiver_body()
1975 		targ->ctr.events++;  in receiver_body()
1979 	int dump = targ->g->options & OPT_DUMP;  in receiver_body()
1981 	nifp = targ->nmd->nifp;  in receiver_body()
1982 	while (!targ->cancel && (n == 0 || targ->ctr.pkts < n)) {  in receiver_body()
1987 			D("ioctl error on queue %d: %s", targ->me,  in receiver_body()
1992 		if (poll(&pfd, 1, 1 * 1000) <= 0 && !targ->g->forever) {  in receiver_body()
1993 			clock_gettime(CLOCK_REALTIME_PRECISE, &targ->toc);  in receiver_body()
1994 			targ->toc.tv_sec -= 1; /* Subtract timeout time. */  in receiver_body()
2004 		for (i = targ->nmd->first_rx_ring; i <= targ->nmd->last_rx_ring; i++) {  in receiver_body()
2016 			m = receive_packets(rxring, targ->g->burst, dump, &cur.bytes);  in receiver_body()
2021 		cur.min_space = targ->ctr.min_space;  in receiver_body()
2024 		targ->ctr = cur;  in receiver_body()
2028 	clock_gettime(CLOCK_REALTIME_PRECISE, &targ->toc);  in receiver_body()
2033 	targ->completed = 1;  in receiver_body()
2034 	targ->ctr = cur;  in receiver_body()
2038 	targ->used = 0;  in receiver_body()
2046 	struct targ *targ = (struct targ *) data;  in txseq_body()  local
2047 	struct pollfd pfd = { .fd = targ->fd, .events = POLLOUT };  in txseq_body()
2051 	int options = targ->g->options | OPT_COPY;  in txseq_body()
2053 	int rate_limit = targ->g->tx_rate;  in txseq_body()
2054 	struct pkt *pkt = &targ->pkt;  in txseq_body()
2055 	int frags = targ->g->frags;  in txseq_body()
2061 	if (targ->g->nthreads > 1) {  in txseq_body()
2066 	if (targ->g->npackets > 0) {  in txseq_body()
2070 	frame = (char *)pkt + sizeof(pkt->vh) - targ->g->virt_header;  in txseq_body()
2071 	size = targ->g->pkt_size + targ->g->virt_header;  in txseq_body()
2073 	D("start, fd %d main_fd %d", targ->fd, targ->g->main_fd);  in txseq_body()
2074 	if (setaffinity(targ->thread, targ->affinity))  in txseq_body()
2077 	clock_gettime(CLOCK_REALTIME_PRECISE, &targ->tic);  in txseq_body()
2079 		targ->tic = timespec_add(targ->tic, (struct timespec){2,0});  in txseq_body()
2080 		targ->tic.tv_nsec = 0;  in txseq_body()
2081 		wait_time(targ->tic);  in txseq_body()
2082 		nexttime = targ->tic;  in txseq_body()
2086 	ring = NETMAP_TXRING(targ->nmd->nifp, targ->nmd->first_tx_ring);  in txseq_body()
2088 	while (!targ->cancel) {  in txseq_body()
2097 			budget = targ->g->burst;  in txseq_body()
2100 			budget = targ->g->burst;  in txseq_body()
2101 			nexttime = timespec_add(nexttime, targ->g->tx_period);  in txseq_body()
2109 			D("ioctl error on queue %d: %s", targ->me,  in txseq_body()
2115 			if (targ->cancel)  in txseq_body()
2117 			D("poll error on queue %d: %s", targ->me,  in txseq_body()
2123 				targ->nmd->first_tx_ring, targ->nmd->last_tx_ring);  in txseq_body()
2151 			uint16_t *w = (uint16_t *)PKT(pkt, body, targ->g->af), t;  in txseq_body()
2153 …	memcpy(&sum, targ->g->af == AF_INET ? &pkt->ipv4.udp.uh_sum : &pkt->ipv6.udp.uh_sum, sizeof(sum));  in txseq_body()
2157 			PKT(pkt, body, targ->g->af)[0] = sequence >> 24;  in txseq_body()
2158 			PKT(pkt, body, targ->g->af)[1] = (sequence >> 16) & 0xff;  in txseq_body()
2161 			PKT(pkt, body, targ->g->af)[2] = (sequence >> 8) & 0xff;  in txseq_body()
2162 			PKT(pkt, body, targ->g->af)[3] = sequence & 0xff;  in txseq_body()
2164 …	memcpy(targ->g->af == AF_INET ? &pkt->ipv4.udp.uh_sum : &pkt->ipv6.udp.uh_sum, &sum, sizeof(sum));  in txseq_body()
2167 				update_addresses(pkt, targ);  in txseq_body()
2198 		targ->ctr.pkts = sent;  in txseq_body()
2199 		targ->ctr.bytes = sent * size;  in txseq_body()
2200 		targ->ctr.events = event;  in txseq_body()
2210 	while (!targ->cancel && nm_tx_pending(ring)) {  in txseq_body()
2212 				ring->tail, ring->head, targ->nmd->first_tx_ring);  in txseq_body()
2217 	clock_gettime(CLOCK_REALTIME_PRECISE, &targ->toc);  in txseq_body()
2218 	targ->completed = 1;  in txseq_body()
2219 	targ->ctr.pkts = sent;  in txseq_body()
2220 	targ->ctr.bytes = sent * size;  in txseq_body()
2221 	targ->ctr.events = event;  in txseq_body()
2224 	targ->used = 0;  in txseq_body()
2256 	struct targ *targ = (struct targ *) data;  in rxseq_body()  local
2257 	struct pollfd pfd = { .fd = targ->fd, .events = POLLIN };  in rxseq_body()
2258 	int dump = targ->g->options & OPT_DUMP;  in rxseq_body()
2270 	if (setaffinity(targ->thread, targ->affinity))  in rxseq_body()
2273 	nrings = targ->nmd->last_rx_ring - targ->nmd->first_rx_ring + 1;  in rxseq_body()
2281 		targ->g->ifname, targ->fd, targ->g->main_fd);  in rxseq_body()
2283 	for (;!targ->cancel;) {  in rxseq_body()
2291 	clock_gettime(CLOCK_REALTIME_PRECISE, &targ->tic);  in rxseq_body()
2294 	while (!targ->cancel) {  in rxseq_body()
2300 			D("ioctl error on queue %d: %s", targ->me,  in rxseq_body()
2305 		if (poll(&pfd, 1, 1 * 1000) <= 0 && !targ->g->forever) {  in rxseq_body()
2306 			clock_gettime(CLOCK_REALTIME_PRECISE, &targ->toc);  in rxseq_body()
2307 			targ->toc.tv_sec -= 1; /* Subtract timeout time. */  in rxseq_body()
2317 		for (j = targ->nmd->first_rx_ring; j <= targ->nmd->last_rx_ring; j++) {  in rxseq_body()
2318 			ring = NETMAP_RXRING(targ->nmd->nifp, j);  in rxseq_body()
2323 			if (limit > targ->g->burst)  in rxseq_body()
2324 				limit = targ->g->burst;  in rxseq_body()
2373 				p -= sizeof(pkt->vh) - targ->g->virt_header;  in rxseq_body()
2374 				len += sizeof(pkt->vh) - targ->g->virt_header;  in rxseq_body()
2418 			targ->ctr = cur;  in rxseq_body()
2421 	clock_gettime(CLOCK_REALTIME_PRECISE, &targ->toc);  in rxseq_body()
2426 	targ->completed = 1;  in rxseq_body()
2427 	targ->ctr = cur;  in rxseq_body()
2433 	targ->used = 0;  in rxseq_body()
2622 	struct targ *t;  in start_threads()