1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
5  *
6  *   Redistribution and use in source and binary forms, with or without
7  *   modification, are permitted provided that the following conditions
8  *   are met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above copyright
13  *       notice, this list of conditions and the following disclaimer in
14  *       the documentation and/or other materials provided with the
15  *       distribution.
16  *     * Neither the name of Intel Corporation nor the names of its
17  *       contributors may be used to endorse or promote products derived
18  *       from this software without specific prior written permission.
19  *
20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #include <stdio.h>
34 #include <sys/queue.h>
35 #include <string.h>
36 #include <rte_mbuf.h>
37 #include <rte_memory.h>
38 #include <rte_cycles.h>
39 #include <rte_compat.h>
40 #include <rte_memzone.h>
41 #include <rte_errno.h>
42 #include <rte_string_fns.h>
43 #include <rte_eal_memconfig.h>
44 #include <rte_pause.h>
45 
46 #include "rte_distributor_private.h"
47 #include "rte_distributor.h"
48 #include "rte_distributor_v20.h"
49 #include "rte_distributor_v1705.h"
50 
51 TAILQ_HEAD(rte_dist_burst_list, rte_distributor);
52 
53 static struct rte_tailq_elem rte_dist_burst_tailq = {
54 	.name = "RTE_DIST_BURST",
55 };
56 EAL_REGISTER_TAILQ(rte_dist_burst_tailq)
57 
58 /**** APIs called by workers ****/
59 
60 /**** Burst Packet APIs called by workers ****/
61 
62 void
63 rte_distributor_request_pkt_v1705(struct rte_distributor *d,
64 		unsigned int worker_id, struct rte_mbuf **oldpkt,
65 		unsigned int count)
66 {
67 	struct rte_distributor_buffer *buf = &(d->bufs[worker_id]);
68 	unsigned int i;
69 
70 	volatile int64_t *retptr64;
71 
72 	if (unlikely(d->alg_type == RTE_DIST_ALG_SINGLE)) {
73 		rte_distributor_request_pkt_v20(d->d_v20,
74 			worker_id, oldpkt[0]);
75 		return;
76 	}
77 
78 	retptr64 = &(buf->retptr64[0]);
79 	/* Spin while handshake bits are set (scheduler clears it) */
80 	while (unlikely(*retptr64 & RTE_DISTRIB_GET_BUF)) {
81 		rte_pause();
82 		uint64_t t = rte_rdtsc()+100;
83 
84 		while (rte_rdtsc() < t)
85 			rte_pause();
86 	}
87 
88 	/*
89 	 * OK, if we've got here, then the scheduler has just cleared the
90 	 * handshake bits. Populate the retptrs with returning packets.
91 	 */
92 
93 	for (i = count; i < RTE_DIST_BURST_SIZE; i++)
94 		buf->retptr64[i] = 0;
95 
96 	/* Set Return bit for each packet returned */
97 	for (i = count; i-- > 0; )
98 		buf->retptr64[i] =
99 			(((int64_t)(uintptr_t)(oldpkt[i])) <<
100 			RTE_DISTRIB_FLAG_BITS) | RTE_DISTRIB_RETURN_BUF;
101 
102 	/*
103 	 * Finally, set the GET_BUF  to signal to distributor that cache
104 	 * line is ready for processing
105 	 */
106 	*retptr64 |= RTE_DISTRIB_GET_BUF;
107 }
108 BIND_DEFAULT_SYMBOL(rte_distributor_request_pkt, _v1705, 17.05);
109 MAP_STATIC_SYMBOL(void rte_distributor_request_pkt(struct rte_distributor *d,
110 		unsigned int worker_id, struct rte_mbuf **oldpkt,
111 		unsigned int count),
112 		rte_distributor_request_pkt_v1705);
113 
114 int
115 rte_distributor_poll_pkt_v1705(struct rte_distributor *d,
116 		unsigned int worker_id, struct rte_mbuf **pkts)
117 {
118 	struct rte_distributor_buffer *buf = &d->bufs[worker_id];
119 	uint64_t ret;
120 	int count = 0;
121 	unsigned int i;
122 
123 	if (unlikely(d->alg_type == RTE_DIST_ALG_SINGLE)) {
124 		pkts[0] = rte_distributor_poll_pkt_v20(d->d_v20, worker_id);
125 		return (pkts[0]) ? 1 : 0;
126 	}
127 
128 	/* If bit is set, return */
129 	if (buf->bufptr64[0] & RTE_DISTRIB_GET_BUF)
130 		return -1;
131 
132 	/* since bufptr64 is signed, this should be an arithmetic shift */
133 	for (i = 0; i < RTE_DIST_BURST_SIZE; i++) {
134 		if (likely(buf->bufptr64[i] & RTE_DISTRIB_VALID_BUF)) {
135 			ret = buf->bufptr64[i] >> RTE_DISTRIB_FLAG_BITS;
136 			pkts[count++] = (struct rte_mbuf *)((uintptr_t)(ret));
137 		}
138 	}
139 
140 	/*
141 	 * so now we've got the contents of the cacheline into an  array of
142 	 * mbuf pointers, so toggle the bit so scheduler can start working
143 	 * on the next cacheline while we're working.
144 	 */
145 	buf->bufptr64[0] |= RTE_DISTRIB_GET_BUF;
146 
147 	return count;
148 }
149 BIND_DEFAULT_SYMBOL(rte_distributor_poll_pkt, _v1705, 17.05);
150 MAP_STATIC_SYMBOL(int rte_distributor_poll_pkt(struct rte_distributor *d,
151 		unsigned int worker_id, struct rte_mbuf **pkts),
152 		rte_distributor_poll_pkt_v1705);
153 
154 int
155 rte_distributor_get_pkt_v1705(struct rte_distributor *d,
156 		unsigned int worker_id, struct rte_mbuf **pkts,
157 		struct rte_mbuf **oldpkt, unsigned int return_count)
158 {
159 	int count;
160 
161 	if (unlikely(d->alg_type == RTE_DIST_ALG_SINGLE)) {
162 		if (return_count <= 1) {
163 			pkts[0] = rte_distributor_get_pkt_v20(d->d_v20,
164 				worker_id, oldpkt[0]);
165 			return (pkts[0]) ? 1 : 0;
166 		} else
167 			return -EINVAL;
168 	}
169 
170 	rte_distributor_request_pkt(d, worker_id, oldpkt, return_count);
171 
172 	count = rte_distributor_poll_pkt(d, worker_id, pkts);
173 	while (count == -1) {
174 		uint64_t t = rte_rdtsc() + 100;
175 
176 		while (rte_rdtsc() < t)
177 			rte_pause();
178 
179 		count = rte_distributor_poll_pkt(d, worker_id, pkts);
180 	}
181 	return count;
182 }
183 BIND_DEFAULT_SYMBOL(rte_distributor_get_pkt, _v1705, 17.05);
184 MAP_STATIC_SYMBOL(int rte_distributor_get_pkt(struct rte_distributor *d,
185 		unsigned int worker_id, struct rte_mbuf **pkts,
186 		struct rte_mbuf **oldpkt, unsigned int return_count),
187 		rte_distributor_get_pkt_v1705);
188 
189 int
190 rte_distributor_return_pkt_v1705(struct rte_distributor *d,
191 		unsigned int worker_id, struct rte_mbuf **oldpkt, int num)
192 {
193 	struct rte_distributor_buffer *buf = &d->bufs[worker_id];
194 	unsigned int i;
195 
196 	if (unlikely(d->alg_type == RTE_DIST_ALG_SINGLE)) {
197 		if (num == 1)
198 			return rte_distributor_return_pkt_v20(d->d_v20,
199 				worker_id, oldpkt[0]);
200 		else
201 			return -EINVAL;
202 	}
203 
204 	for (i = 0; i < RTE_DIST_BURST_SIZE; i++)
205 		/* Switch off the return bit first */
206 		buf->retptr64[i] &= ~RTE_DISTRIB_RETURN_BUF;
207 
208 	for (i = num; i-- > 0; )
209 		buf->retptr64[i] = (((int64_t)(uintptr_t)oldpkt[i]) <<
210 			RTE_DISTRIB_FLAG_BITS) | RTE_DISTRIB_RETURN_BUF;
211 
212 	/* set the GET_BUF but even if we got no returns */
213 	buf->retptr64[0] |= RTE_DISTRIB_GET_BUF;
214 
215 	return 0;
216 }
217 BIND_DEFAULT_SYMBOL(rte_distributor_return_pkt, _v1705, 17.05);
218 MAP_STATIC_SYMBOL(int rte_distributor_return_pkt(struct rte_distributor *d,
219 		unsigned int worker_id, struct rte_mbuf **oldpkt, int num),
220 		rte_distributor_return_pkt_v1705);
221 
222 /**** APIs called on distributor core ***/
223 
224 /* stores a packet returned from a worker inside the returns array */
225 static inline void
226 store_return(uintptr_t oldbuf, struct rte_distributor *d,
227 		unsigned int *ret_start, unsigned int *ret_count)
228 {
229 	if (!oldbuf)
230 		return;
231 	/* store returns in a circular buffer */
232 	d->returns.mbufs[(*ret_start + *ret_count) & RTE_DISTRIB_RETURNS_MASK]
233 			= (void *)oldbuf;
234 	*ret_start += (*ret_count == RTE_DISTRIB_RETURNS_MASK);
235 	*ret_count += (*ret_count != RTE_DISTRIB_RETURNS_MASK);
236 }
237 
238 /*
239  * Match then flow_ids (tags) of the incoming packets to the flow_ids
240  * of the inflight packets (both inflight on the workers and in each worker
241  * backlog). This will then allow us to pin those packets to the relevant
242  * workers to give us our atomic flow pinning.
243  */
244 void
245 find_match_scalar(struct rte_distributor *d,
246 			uint16_t *data_ptr,
247 			uint16_t *output_ptr)
248 {
249 	struct rte_distributor_backlog *bl;
250 	uint16_t i, j, w;
251 
252 	/*
253 	 * Function overview:
254 	 * 1. Loop through all worker ID's
255 	 * 2. Compare the current inflights to the incoming tags
256 	 * 3. Compare the current backlog to the incoming tags
257 	 * 4. Add any matches to the output
258 	 */
259 
260 	for (j = 0 ; j < RTE_DIST_BURST_SIZE; j++)
261 		output_ptr[j] = 0;
262 
263 	for (i = 0; i < d->num_workers; i++) {
264 		bl = &d->backlog[i];
265 
266 		for (j = 0; j < RTE_DIST_BURST_SIZE ; j++)
267 			for (w = 0; w < RTE_DIST_BURST_SIZE; w++)
268 				if (d->in_flight_tags[i][j] == data_ptr[w]) {
269 					output_ptr[j] = i+1;
270 					break;
271 				}
272 		for (j = 0; j < RTE_DIST_BURST_SIZE; j++)
273 			for (w = 0; w < RTE_DIST_BURST_SIZE; w++)
274 				if (bl->tags[j] == data_ptr[w]) {
275 					output_ptr[j] = i+1;
276 					break;
277 				}
278 	}
279 
280 	/*
281 	 * At this stage, the output contains 8 16-bit values, with
282 	 * each non-zero value containing the worker ID on which the
283 	 * corresponding flow is pinned to.
284 	 */
285 }
286 
287 
288 /*
289  * When the handshake bits indicate that there are packets coming
290  * back from the worker, this function is called to copy and store
291  * the valid returned pointers (store_return).
292  */
293 static unsigned int
294 handle_returns(struct rte_distributor *d, unsigned int wkr)
295 {
296 	struct rte_distributor_buffer *buf = &(d->bufs[wkr]);
297 	uintptr_t oldbuf;
298 	unsigned int ret_start = d->returns.start,
299 			ret_count = d->returns.count;
300 	unsigned int count = 0;
301 	unsigned int i;
302 
303 	if (buf->retptr64[0] & RTE_DISTRIB_GET_BUF) {
304 		for (i = 0; i < RTE_DIST_BURST_SIZE; i++) {
305 			if (buf->retptr64[i] & RTE_DISTRIB_RETURN_BUF) {
306 				oldbuf = ((uintptr_t)(buf->retptr64[i] >>
307 					RTE_DISTRIB_FLAG_BITS));
308 				/* store returns in a circular buffer */
309 				store_return(oldbuf, d, &ret_start, &ret_count);
310 				count++;
311 				buf->retptr64[i] &= ~RTE_DISTRIB_RETURN_BUF;
312 			}
313 		}
314 		d->returns.start = ret_start;
315 		d->returns.count = ret_count;
316 		/* Clear for the worker to populate with more returns */
317 		buf->retptr64[0] = 0;
318 	}
319 	return count;
320 }
321 
322 /*
323  * This function releases a burst (cache line) to a worker.
324  * It is called from the process function when a cacheline is
325  * full to make room for more packets for that worker, or when
326  * all packets have been assigned to bursts and need to be flushed
327  * to the workers.
328  * It also needs to wait for any outstanding packets from the worker
329  * before sending out new packets.
330  */
331 static unsigned int
332 release(struct rte_distributor *d, unsigned int wkr)
333 {
334 	struct rte_distributor_buffer *buf = &(d->bufs[wkr]);
335 	unsigned int i;
336 
337 	while (!(d->bufs[wkr].bufptr64[0] & RTE_DISTRIB_GET_BUF))
338 		rte_pause();
339 
340 	handle_returns(d, wkr);
341 
342 	buf->count = 0;
343 
344 	for (i = 0; i < d->backlog[wkr].count; i++) {
345 		d->bufs[wkr].bufptr64[i] = d->backlog[wkr].pkts[i] |
346 				RTE_DISTRIB_GET_BUF | RTE_DISTRIB_VALID_BUF;
347 		d->in_flight_tags[wkr][i] = d->backlog[wkr].tags[i];
348 	}
349 	buf->count = i;
350 	for ( ; i < RTE_DIST_BURST_SIZE ; i++) {
351 		buf->bufptr64[i] = RTE_DISTRIB_GET_BUF;
352 		d->in_flight_tags[wkr][i] = 0;
353 	}
354 
355 	d->backlog[wkr].count = 0;
356 
357 	/* Clear the GET bit */
358 	buf->bufptr64[0] &= ~RTE_DISTRIB_GET_BUF;
359 	return  buf->count;
360 
361 }
362 
363 
364 /* process a set of packets to distribute them to workers */
365 int
366 rte_distributor_process_v1705(struct rte_distributor *d,
367 		struct rte_mbuf **mbufs, unsigned int num_mbufs)
368 {
369 	unsigned int next_idx = 0;
370 	static unsigned int wkr;
371 	struct rte_mbuf *next_mb = NULL;
372 	int64_t next_value = 0;
373 	uint16_t new_tag = 0;
374 	uint16_t flows[RTE_DIST_BURST_SIZE] __rte_cache_aligned;
375 	unsigned int i, j, w, wid;
376 
377 	if (d->alg_type == RTE_DIST_ALG_SINGLE) {
378 		/* Call the old API */
379 		return rte_distributor_process_v20(d->d_v20, mbufs, num_mbufs);
380 	}
381 
382 	if (unlikely(num_mbufs == 0)) {
383 		/* Flush out all non-full cache-lines to workers. */
384 		for (wid = 0 ; wid < d->num_workers; wid++) {
385 			if ((d->bufs[wid].bufptr64[0] & RTE_DISTRIB_GET_BUF)) {
386 				release(d, wid);
387 				handle_returns(d, wid);
388 			}
389 		}
390 		return 0;
391 	}
392 
393 	while (next_idx < num_mbufs) {
394 		uint16_t matches[RTE_DIST_BURST_SIZE];
395 		unsigned int pkts;
396 
397 		if (d->bufs[wkr].bufptr64[0] & RTE_DISTRIB_GET_BUF)
398 			d->bufs[wkr].count = 0;
399 
400 		if ((num_mbufs - next_idx) < RTE_DIST_BURST_SIZE)
401 			pkts = num_mbufs - next_idx;
402 		else
403 			pkts = RTE_DIST_BURST_SIZE;
404 
405 		for (i = 0; i < pkts; i++) {
406 			if (mbufs[next_idx + i]) {
407 				/* flows have to be non-zero */
408 				flows[i] = mbufs[next_idx + i]->hash.usr | 1;
409 			} else
410 				flows[i] = 0;
411 		}
412 		for (; i < RTE_DIST_BURST_SIZE; i++)
413 			flows[i] = 0;
414 
415 		switch (d->dist_match_fn) {
416 		case RTE_DIST_MATCH_VECTOR:
417 			find_match_vec(d, &flows[0], &matches[0]);
418 			break;
419 		default:
420 			find_match_scalar(d, &flows[0], &matches[0]);
421 		}
422 
423 		/*
424 		 * Matches array now contain the intended worker ID (+1) of
425 		 * the incoming packets. Any zeroes need to be assigned
426 		 * workers.
427 		 */
428 
429 		for (j = 0; j < pkts; j++) {
430 
431 			next_mb = mbufs[next_idx++];
432 			next_value = (((int64_t)(uintptr_t)next_mb) <<
433 					RTE_DISTRIB_FLAG_BITS);
434 			/*
435 			 * User is advocated to set tag value for each
436 			 * mbuf before calling rte_distributor_process.
437 			 * User defined tags are used to identify flows,
438 			 * or sessions.
439 			 */
440 			/* flows MUST be non-zero */
441 			new_tag = (uint16_t)(next_mb->hash.usr) | 1;
442 
443 			/*
444 			 * Uncommenting the next line will cause the find_match
445 			 * function to be optimized out, making this function
446 			 * do parallel (non-atomic) distribution
447 			 */
448 			/* matches[j] = 0; */
449 
450 			if (matches[j]) {
451 				struct rte_distributor_backlog *bl =
452 						&d->backlog[matches[j]-1];
453 				if (unlikely(bl->count ==
454 						RTE_DIST_BURST_SIZE)) {
455 					release(d, matches[j]-1);
456 				}
457 
458 				/* Add to worker that already has flow */
459 				unsigned int idx = bl->count++;
460 
461 				bl->tags[idx] = new_tag;
462 				bl->pkts[idx] = next_value;
463 
464 			} else {
465 				struct rte_distributor_backlog *bl =
466 						&d->backlog[wkr];
467 				if (unlikely(bl->count ==
468 						RTE_DIST_BURST_SIZE)) {
469 					release(d, wkr);
470 				}
471 
472 				/* Add to current worker worker */
473 				unsigned int idx = bl->count++;
474 
475 				bl->tags[idx] = new_tag;
476 				bl->pkts[idx] = next_value;
477 				/*
478 				 * Now that we've just added an unpinned flow
479 				 * to a worker, we need to ensure that all
480 				 * other packets with that same flow will go
481 				 * to the same worker in this burst.
482 				 */
483 				for (w = j; w < pkts; w++)
484 					if (flows[w] == new_tag)
485 						matches[w] = wkr+1;
486 			}
487 		}
488 		wkr++;
489 		if (wkr >= d->num_workers)
490 			wkr = 0;
491 	}
492 
493 	/* Flush out all non-full cache-lines to workers. */
494 	for (wid = 0 ; wid < d->num_workers; wid++)
495 		if ((d->bufs[wid].bufptr64[0] & RTE_DISTRIB_GET_BUF))
496 			release(d, wid);
497 
498 	return num_mbufs;
499 }
500 BIND_DEFAULT_SYMBOL(rte_distributor_process, _v1705, 17.05);
501 MAP_STATIC_SYMBOL(int rte_distributor_process(struct rte_distributor *d,
502 		struct rte_mbuf **mbufs, unsigned int num_mbufs),
503 		rte_distributor_process_v1705);
504 
505 /* return to the caller, packets returned from workers */
506 int
507 rte_distributor_returned_pkts_v1705(struct rte_distributor *d,
508 		struct rte_mbuf **mbufs, unsigned int max_mbufs)
509 {
510 	struct rte_distributor_returned_pkts *returns = &d->returns;
511 	unsigned int retval = (max_mbufs < returns->count) ?
512 			max_mbufs : returns->count;
513 	unsigned int i;
514 
515 	if (d->alg_type == RTE_DIST_ALG_SINGLE) {
516 		/* Call the old API */
517 		return rte_distributor_returned_pkts_v20(d->d_v20,
518 				mbufs, max_mbufs);
519 	}
520 
521 	for (i = 0; i < retval; i++) {
522 		unsigned int idx = (returns->start + i) &
523 				RTE_DISTRIB_RETURNS_MASK;
524 
525 		mbufs[i] = returns->mbufs[idx];
526 	}
527 	returns->start += i;
528 	returns->count -= i;
529 
530 	return retval;
531 }
532 BIND_DEFAULT_SYMBOL(rte_distributor_returned_pkts, _v1705, 17.05);
533 MAP_STATIC_SYMBOL(int rte_distributor_returned_pkts(struct rte_distributor *d,
534 		struct rte_mbuf **mbufs, unsigned int max_mbufs),
535 		rte_distributor_returned_pkts_v1705);
536 
537 /*
538  * Return the number of packets in-flight in a distributor, i.e. packets
539  * being worked on or queued up in a backlog.
540  */
541 static inline unsigned int
542 total_outstanding(const struct rte_distributor *d)
543 {
544 	unsigned int wkr, total_outstanding = 0;
545 
546 	for (wkr = 0; wkr < d->num_workers; wkr++)
547 		total_outstanding += d->backlog[wkr].count;
548 
549 	return total_outstanding;
550 }
551 
552 /*
553  * Flush the distributor, so that there are no outstanding packets in flight or
554  * queued up.
555  */
556 int
557 rte_distributor_flush_v1705(struct rte_distributor *d)
558 {
559 	unsigned int flushed;
560 	unsigned int wkr;
561 
562 	if (d->alg_type == RTE_DIST_ALG_SINGLE) {
563 		/* Call the old API */
564 		return rte_distributor_flush_v20(d->d_v20);
565 	}
566 
567 	flushed = total_outstanding(d);
568 
569 	while (total_outstanding(d) > 0)
570 		rte_distributor_process(d, NULL, 0);
571 
572 	/*
573 	 * Send empty burst to all workers to allow them to exit
574 	 * gracefully, should they need to.
575 	 */
576 	rte_distributor_process(d, NULL, 0);
577 
578 	for (wkr = 0; wkr < d->num_workers; wkr++)
579 		handle_returns(d, wkr);
580 
581 	return flushed;
582 }
583 BIND_DEFAULT_SYMBOL(rte_distributor_flush, _v1705, 17.05);
584 MAP_STATIC_SYMBOL(int rte_distributor_flush(struct rte_distributor *d),
585 		rte_distributor_flush_v1705);
586 
587 /* clears the internal returns array in the distributor */
588 void
589 rte_distributor_clear_returns_v1705(struct rte_distributor *d)
590 {
591 	unsigned int wkr;
592 
593 	if (d->alg_type == RTE_DIST_ALG_SINGLE) {
594 		/* Call the old API */
595 		rte_distributor_clear_returns_v20(d->d_v20);
596 		return;
597 	}
598 
599 	/* throw away returns, so workers can exit */
600 	for (wkr = 0; wkr < d->num_workers; wkr++)
601 		d->bufs[wkr].retptr64[0] = 0;
602 }
603 BIND_DEFAULT_SYMBOL(rte_distributor_clear_returns, _v1705, 17.05);
604 MAP_STATIC_SYMBOL(void rte_distributor_clear_returns(struct rte_distributor *d),
605 		rte_distributor_clear_returns_v1705);
606 
607 /* creates a distributor instance */
608 struct rte_distributor *
609 rte_distributor_create_v1705(const char *name,
610 		unsigned int socket_id,
611 		unsigned int num_workers,
612 		unsigned int alg_type)
613 {
614 	struct rte_distributor *d;
615 	struct rte_dist_burst_list *dist_burst_list;
616 	char mz_name[RTE_MEMZONE_NAMESIZE];
617 	const struct rte_memzone *mz;
618 	unsigned int i;
619 
620 	/* TODO Reorganise function properly around RTE_DIST_ALG_SINGLE/BURST */
621 
622 	/* compilation-time checks */
623 	RTE_BUILD_BUG_ON((sizeof(*d) & RTE_CACHE_LINE_MASK) != 0);
624 	RTE_BUILD_BUG_ON((RTE_DISTRIB_MAX_WORKERS & 7) != 0);
625 
626 	if (alg_type == RTE_DIST_ALG_SINGLE) {
627 		d = malloc(sizeof(struct rte_distributor));
628 		if (d == NULL) {
629 			rte_errno = ENOMEM;
630 			return NULL;
631 		}
632 		d->d_v20 = rte_distributor_create_v20(name,
633 				socket_id, num_workers);
634 		if (d->d_v20 == NULL) {
635 			free(d);
636 			/* rte_errno will have been set */
637 			return NULL;
638 		}
639 		d->alg_type = alg_type;
640 		return d;
641 	}
642 
643 	if (name == NULL || num_workers >= RTE_DISTRIB_MAX_WORKERS) {
644 		rte_errno = EINVAL;
645 		return NULL;
646 	}
647 
648 	snprintf(mz_name, sizeof(mz_name), RTE_DISTRIB_PREFIX"%s", name);
649 	mz = rte_memzone_reserve(mz_name, sizeof(*d), socket_id, NO_FLAGS);
650 	if (mz == NULL) {
651 		rte_errno = ENOMEM;
652 		return NULL;
653 	}
654 
655 	d = mz->addr;
656 	snprintf(d->name, sizeof(d->name), "%s", name);
657 	d->num_workers = num_workers;
658 	d->alg_type = alg_type;
659 
660 	d->dist_match_fn = RTE_DIST_MATCH_SCALAR;
661 #if defined(RTE_ARCH_X86)
662 	d->dist_match_fn = RTE_DIST_MATCH_VECTOR;
663 #endif
664 
665 	/*
666 	 * Set up the backlog tags so they're pointing at the second cache
667 	 * line for performance during flow matching
668 	 */
669 	for (i = 0 ; i < num_workers ; i++)
670 		d->backlog[i].tags = &d->in_flight_tags[i][RTE_DIST_BURST_SIZE];
671 
672 	dist_burst_list = RTE_TAILQ_CAST(rte_dist_burst_tailq.head,
673 					  rte_dist_burst_list);
674 
675 
676 	rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
677 	TAILQ_INSERT_TAIL(dist_burst_list, d, next);
678 	rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
679 
680 	return d;
681 }
682 BIND_DEFAULT_SYMBOL(rte_distributor_create, _v1705, 17.05);
683 MAP_STATIC_SYMBOL(struct rte_distributor *rte_distributor_create(
684 		const char *name, unsigned int socket_id,
685 		unsigned int num_workers, unsigned int alg_type),
686 		rte_distributor_create_v1705);
687