1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
3 */
4
5 #include <rte_mbuf.h>
6 #include "rte_distributor.h"
7 #include "distributor_private.h"
8 #include "smmintrin.h"
9 #include "nmmintrin.h"
10
11
12 void
find_match_vec(struct rte_distributor * d,uint16_t * data_ptr,uint16_t * output_ptr)13 find_match_vec(struct rte_distributor *d,
14 uint16_t *data_ptr,
15 uint16_t *output_ptr)
16 {
17 /* Setup */
18 __m128i incoming_fids;
19 __m128i inflight_fids;
20 __m128i preflight_fids;
21 __m128i wkr;
22 __m128i mask1;
23 __m128i mask2;
24 __m128i output;
25 struct rte_distributor_backlog *bl;
26 uint16_t i;
27
28 /*
29 * Function overview:
30 * 2. Loop through all worker ID's
31 * 2a. Load the current inflights for that worker into an xmm reg
32 * 2b. Load the current backlog for that worker into an xmm reg
33 * 2c. use cmpestrm to intersect flow_ids with backlog and inflights
34 * 2d. Add any matches to the output
35 * 3. Write the output xmm (matching worker ids).
36 */
37
38
39 output = _mm_set1_epi16(0);
40 incoming_fids = _mm_load_si128((__m128i *)data_ptr);
41
42 for (i = 0; i < d->num_workers; i++) {
43 bl = &d->backlog[i];
44
45 inflight_fids =
46 _mm_load_si128((__m128i *)&(d->in_flight_tags[i]));
47 preflight_fids =
48 _mm_load_si128((__m128i *)(bl->tags));
49
50 /*
51 * Any incoming_fid that exists anywhere in inflight_fids will
52 * have 0xffff in same position of the mask as the incoming fid
53 * Example (shortened to bytes for brevity):
54 * incoming_fids 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08
55 * inflight_fids 0x03 0x05 0x07 0x00 0x00 0x00 0x00 0x00
56 * mask 0x00 0x00 0xff 0x00 0xff 0x00 0xff 0x00
57 */
58
59 mask1 = _mm_cmpestrm(inflight_fids, 8, incoming_fids, 8,
60 _SIDD_UWORD_OPS |
61 _SIDD_CMP_EQUAL_ANY |
62 _SIDD_UNIT_MASK);
63 mask2 = _mm_cmpestrm(preflight_fids, 8, incoming_fids, 8,
64 _SIDD_UWORD_OPS |
65 _SIDD_CMP_EQUAL_ANY |
66 _SIDD_UNIT_MASK);
67
68 mask1 = _mm_or_si128(mask1, mask2);
69 /*
70 * Now mask contains 0xffff where there's a match.
71 * Next we need to store the worker_id in the relevant position
72 * in the output.
73 */
74
75 wkr = _mm_set1_epi16(i+1);
76 mask1 = _mm_and_si128(mask1, wkr);
77 output = _mm_or_si128(mask1, output);
78 }
79
80 /*
81 * At this stage, the output 128-bit contains 8 16-bit values, with
82 * each non-zero value containing the worker ID on which the
83 * corresponding flow is pinned to.
84 */
85 _mm_store_si128((__m128i *)output_ptr, output);
86 }
87