1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2019 Mellanox Technologies, Ltd
3 */
4 #include <netinet/in.h>
5
6 #include <rte_malloc.h>
7 #include <rte_errno.h>
8 #include <rte_common.h>
9
10 #include <mlx5_common.h>
11
12 #include "mlx5_vdpa_utils.h"
13 #include "mlx5_vdpa.h"
14
15 static void
mlx5_vdpa_rss_flows_destroy(struct mlx5_vdpa_priv * priv)16 mlx5_vdpa_rss_flows_destroy(struct mlx5_vdpa_priv *priv)
17 {
18 unsigned i;
19
20 for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
21 if (priv->steer.rss[i].flow) {
22 claim_zero(mlx5_glue->dv_destroy_flow
23 (priv->steer.rss[i].flow));
24 priv->steer.rss[i].flow = NULL;
25 }
26 if (priv->steer.rss[i].tir_action) {
27 claim_zero(mlx5_glue->destroy_flow_action
28 (priv->steer.rss[i].tir_action));
29 priv->steer.rss[i].tir_action = NULL;
30 }
31 if (priv->steer.rss[i].tir) {
32 claim_zero(mlx5_devx_cmd_destroy
33 (priv->steer.rss[i].tir));
34 priv->steer.rss[i].tir = NULL;
35 }
36 if (priv->steer.rss[i].matcher) {
37 claim_zero(mlx5_glue->dv_destroy_flow_matcher
38 (priv->steer.rss[i].matcher));
39 priv->steer.rss[i].matcher = NULL;
40 }
41 }
42 }
43
44 void
mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv * priv)45 mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv)
46 {
47 mlx5_vdpa_rss_flows_destroy(priv);
48 if (priv->steer.rqt) {
49 claim_zero(mlx5_devx_cmd_destroy(priv->steer.rqt));
50 priv->steer.rqt = NULL;
51 }
52 }
53
54 #define MLX5_VDPA_DEFAULT_RQT_SIZE 512
55 /*
56 * Return the number of queues configured to the table on success, otherwise
57 * -1 on error.
58 */
59 static int
mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv * priv)60 mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv *priv)
61 {
62 int i;
63 uint32_t rqt_n = RTE_MIN(MLX5_VDPA_DEFAULT_RQT_SIZE,
64 1 << priv->log_max_rqt_size);
65 struct mlx5_devx_rqt_attr *attr = rte_zmalloc(__func__, sizeof(*attr)
66 + rqt_n *
67 sizeof(uint32_t), 0);
68 uint32_t k = 0, j;
69 int ret = 0, num;
70
71 if (!attr) {
72 DRV_LOG(ERR, "Failed to allocate RQT attributes memory.");
73 rte_errno = ENOMEM;
74 return -ENOMEM;
75 }
76 for (i = 0; i < priv->nr_virtqs; i++) {
77 if (is_virtq_recvq(i, priv->nr_virtqs) &&
78 priv->virtqs[i].enable && priv->virtqs[i].virtq) {
79 attr->rq_list[k] = priv->virtqs[i].virtq->id;
80 k++;
81 }
82 }
83 if (k == 0)
84 /* No enabled RQ to configure for RSS. */
85 return 0;
86 num = (int)k;
87 for (j = 0; k != rqt_n; ++k, ++j)
88 attr->rq_list[k] = attr->rq_list[j];
89 attr->rq_type = MLX5_INLINE_Q_TYPE_VIRTQ;
90 attr->rqt_max_size = rqt_n;
91 attr->rqt_actual_size = rqt_n;
92 if (!priv->steer.rqt) {
93 priv->steer.rqt = mlx5_devx_cmd_create_rqt(priv->cdev->ctx,
94 attr);
95 if (!priv->steer.rqt) {
96 DRV_LOG(ERR, "Failed to create RQT.");
97 ret = -rte_errno;
98 }
99 } else {
100 ret = mlx5_devx_cmd_modify_rqt(priv->steer.rqt, attr);
101 if (ret)
102 DRV_LOG(ERR, "Failed to modify RQT.");
103 }
104 rte_free(attr);
105 return ret ? -1 : num;
106 }
107
108 static int __rte_unused
mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv * priv)109 mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv)
110 {
111 #ifdef HAVE_MLX5DV_DR
112 struct mlx5_devx_tir_attr tir_att = {
113 .disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT,
114 .rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ,
115 .transport_domain = priv->td->id,
116 .indirect_table = priv->steer.rqt->id,
117 .rx_hash_symmetric = 1,
118 .rx_hash_toeplitz_key = { 0x2c, 0xc6, 0x81, 0xd1,
119 0x5b, 0xdb, 0xf4, 0xf7,
120 0xfc, 0xa2, 0x83, 0x19,
121 0xdb, 0x1a, 0x3e, 0x94,
122 0x6b, 0x9e, 0x38, 0xd9,
123 0x2c, 0x9c, 0x03, 0xd1,
124 0xad, 0x99, 0x44, 0xa7,
125 0xd9, 0x56, 0x3d, 0x59,
126 0x06, 0x3c, 0x25, 0xf3,
127 0xfc, 0x1f, 0xdc, 0x2a },
128 };
129 struct {
130 size_t size;
131 /**< Size of match value. Do NOT split size and key! */
132 uint32_t buf[MLX5_ST_SZ_DW(fte_match_param)];
133 /**< Matcher value. This value is used as the mask or a key. */
134 } matcher_mask = {
135 .size = sizeof(matcher_mask.buf) -
136 MLX5_ST_SZ_BYTES(fte_match_set_misc4) -
137 MLX5_ST_SZ_BYTES(fte_match_set_misc5),
138 },
139 matcher_value = {
140 .size = sizeof(matcher_value.buf) -
141 MLX5_ST_SZ_BYTES(fte_match_set_misc4) -
142 MLX5_ST_SZ_BYTES(fte_match_set_misc5),
143 };
144 struct mlx5dv_flow_matcher_attr dv_attr = {
145 .type = IBV_FLOW_ATTR_NORMAL,
146 .match_mask = (void *)&matcher_mask,
147 };
148 void *match_m = matcher_mask.buf;
149 void *match_v = matcher_value.buf;
150 void *headers_m = MLX5_ADDR_OF(fte_match_param, match_m, outer_headers);
151 void *headers_v = MLX5_ADDR_OF(fte_match_param, match_v, outer_headers);
152 void *actions[1];
153 const uint8_t l3_hash =
154 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) |
155 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP);
156 const uint8_t l4_hash =
157 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT) |
158 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT);
159 enum { PRIO, CRITERIA, IP_VER_M, IP_VER_V, IP_PROT_M, IP_PROT_V, L3_BIT,
160 L4_BIT, HASH, END};
161 const uint8_t vars[RTE_DIM(priv->steer.rss)][END] = {
162 { 7, 0, 0, 0, 0, 0, 0, 0, 0 },
163 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0, 0,
164 MLX5_L3_PROT_TYPE_IPV4, 0, l3_hash },
165 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0, 0,
166 MLX5_L3_PROT_TYPE_IPV6, 0, l3_hash },
167 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
168 IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_UDP,
169 l3_hash | l4_hash },
170 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
171 IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_TCP,
172 l3_hash | l4_hash },
173 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
174 IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_UDP,
175 l3_hash | l4_hash },
176 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
177 IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_TCP,
178 l3_hash | l4_hash },
179 };
180 unsigned i;
181
182 for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
183 dv_attr.priority = vars[i][PRIO];
184 dv_attr.match_criteria_enable = vars[i][CRITERIA];
185 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
186 vars[i][IP_VER_M]);
187 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
188 vars[i][IP_VER_V]);
189 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
190 vars[i][IP_PROT_M]);
191 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
192 vars[i][IP_PROT_V]);
193 tir_att.rx_hash_field_selector_outer.l3_prot_type =
194 vars[i][L3_BIT];
195 tir_att.rx_hash_field_selector_outer.l4_prot_type =
196 vars[i][L4_BIT];
197 tir_att.rx_hash_field_selector_outer.selected_fields =
198 vars[i][HASH];
199 priv->steer.rss[i].matcher = mlx5_glue->dv_create_flow_matcher
200 (priv->cdev->ctx, &dv_attr, priv->steer.tbl);
201 if (!priv->steer.rss[i].matcher) {
202 DRV_LOG(ERR, "Failed to create matcher %d.", i);
203 goto error;
204 }
205 priv->steer.rss[i].tir = mlx5_devx_cmd_create_tir
206 (priv->cdev->ctx, &tir_att);
207 if (!priv->steer.rss[i].tir) {
208 DRV_LOG(ERR, "Failed to create TIR %d.", i);
209 goto error;
210 }
211 priv->steer.rss[i].tir_action =
212 mlx5_glue->dv_create_flow_action_dest_devx_tir
213 (priv->steer.rss[i].tir->obj);
214 if (!priv->steer.rss[i].tir_action) {
215 DRV_LOG(ERR, "Failed to create TIR action %d.", i);
216 goto error;
217 }
218 actions[0] = priv->steer.rss[i].tir_action;
219 priv->steer.rss[i].flow = mlx5_glue->dv_create_flow
220 (priv->steer.rss[i].matcher,
221 (void *)&matcher_value, 1, actions);
222 if (!priv->steer.rss[i].flow) {
223 DRV_LOG(ERR, "Failed to create flow %d.", i);
224 goto error;
225 }
226 }
227 return 0;
228 error:
229 /* Resources will be freed by the caller. */
230 return -1;
231 #else
232 (void)priv;
233 return -ENOTSUP;
234 #endif /* HAVE_MLX5DV_DR */
235 }
236
237 int
mlx5_vdpa_steer_update(struct mlx5_vdpa_priv * priv)238 mlx5_vdpa_steer_update(struct mlx5_vdpa_priv *priv)
239 {
240 int ret = mlx5_vdpa_rqt_prepare(priv);
241
242 if (ret == 0) {
243 mlx5_vdpa_steer_unset(priv);
244 } else if (ret < 0) {
245 return ret;
246 } else if (!priv->steer.rss[0].flow) {
247 ret = mlx5_vdpa_rss_flows_create(priv);
248 if (ret) {
249 DRV_LOG(ERR, "Cannot create RSS flows.");
250 return -1;
251 }
252 }
253 return 0;
254 }
255
256 int
mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv * priv)257 mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv)
258 {
259 if (mlx5_vdpa_steer_update(priv))
260 goto error;
261 return 0;
262 error:
263 mlx5_vdpa_steer_unset(priv);
264 return -1;
265 }
266