1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */
3
4 #include <linux/refcount.h>
5
6 #include "en_tc.h"
7 #include "en/tc_priv.h"
8 #include "en/tc_ct.h"
9 #include "en/tc/ct_fs.h"
10
11 #include "lib/smfs.h"
12
13 #define INIT_ERR_PREFIX "ct_fs_smfs init failed"
14 #define ct_dbg(fmt, args...)\
15 netdev_dbg(fs->netdev, "ct_fs_smfs debug: " fmt "\n", ##args)
16
17 struct mlx5_ct_fs_smfs_matcher {
18 struct mlx5dr_matcher *dr_matcher;
19 struct list_head list;
20 int prio;
21 refcount_t ref;
22 };
23
24 struct mlx5_ct_fs_smfs_matchers {
25 struct mlx5_ct_fs_smfs_matcher smfs_matchers[6];
26 struct list_head used;
27 };
28
29 struct mlx5_ct_fs_smfs {
30 struct mlx5dr_table *ct_tbl, *ct_nat_tbl;
31 struct mlx5_ct_fs_smfs_matchers matchers;
32 struct mlx5_ct_fs_smfs_matchers matchers_nat;
33 struct mlx5dr_action *fwd_action;
34 struct mlx5_flow_table *ct_nat;
35 struct mutex lock; /* Guards matchers */
36 };
37
38 struct mlx5_ct_fs_smfs_rule {
39 struct mlx5_ct_fs_rule fs_rule;
40 struct mlx5dr_rule *rule;
41 struct mlx5dr_action *count_action;
42 struct mlx5_ct_fs_smfs_matcher *smfs_matcher;
43 };
44
45 static inline void
mlx5_ct_fs_smfs_fill_mask(struct mlx5_ct_fs * fs,struct mlx5_flow_spec * spec,bool ipv4,bool tcp,bool gre)46 mlx5_ct_fs_smfs_fill_mask(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, bool ipv4, bool tcp,
47 bool gre)
48 {
49 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers);
50
51 if (likely(MLX5_CAP_FLOWTABLE_NIC_RX(fs->dev, ft_field_support.outer_ip_version)))
52 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_version);
53 else
54 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype);
55
56 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
57 if (likely(ipv4)) {
58 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c,
59 src_ipv4_src_ipv6.ipv4_layout.ipv4);
60 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c,
61 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
62 } else {
63 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
64 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
65 0xFF,
66 MLX5_FLD_SZ_BYTES(fte_match_set_lyr_2_4,
67 dst_ipv4_dst_ipv6.ipv6_layout.ipv6));
68 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
69 src_ipv4_src_ipv6.ipv6_layout.ipv6),
70 0xFF,
71 MLX5_FLD_SZ_BYTES(fte_match_set_lyr_2_4,
72 src_ipv4_src_ipv6.ipv6_layout.ipv6));
73 }
74
75 if (likely(tcp)) {
76 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, tcp_sport);
77 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, tcp_dport);
78 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags,
79 ntohs(MLX5_CT_TCP_FLAGS_MASK));
80 } else if (!gre) {
81 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, udp_sport);
82 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, udp_dport);
83 }
84
85 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, 0, MLX5_CT_ZONE_MASK);
86 }
87
88 static struct mlx5dr_matcher *
mlx5_ct_fs_smfs_matcher_create(struct mlx5_ct_fs * fs,struct mlx5dr_table * tbl,bool ipv4,bool tcp,bool gre,u32 priority)89 mlx5_ct_fs_smfs_matcher_create(struct mlx5_ct_fs *fs, struct mlx5dr_table *tbl, bool ipv4,
90 bool tcp, bool gre, u32 priority)
91 {
92 struct mlx5dr_matcher *dr_matcher;
93 struct mlx5_flow_spec *spec;
94
95 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
96 if (!spec)
97 return ERR_PTR(-ENOMEM);
98
99 mlx5_ct_fs_smfs_fill_mask(fs, spec, ipv4, tcp, gre);
100 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2 | MLX5_MATCH_OUTER_HEADERS;
101
102 dr_matcher = mlx5_smfs_matcher_create(tbl, priority, spec);
103 kvfree(spec);
104 if (!dr_matcher)
105 return ERR_PTR(-EINVAL);
106
107 return dr_matcher;
108 }
109
110 static struct mlx5_ct_fs_smfs_matcher *
mlx5_ct_fs_smfs_matcher_get(struct mlx5_ct_fs * fs,bool nat,bool ipv4,bool tcp,bool gre)111 mlx5_ct_fs_smfs_matcher_get(struct mlx5_ct_fs *fs, bool nat, bool ipv4, bool tcp, bool gre)
112 {
113 struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
114 struct mlx5_ct_fs_smfs_matcher *m, *smfs_matcher;
115 struct mlx5_ct_fs_smfs_matchers *matchers;
116 struct mlx5dr_matcher *dr_matcher;
117 struct mlx5dr_table *tbl;
118 struct list_head *prev;
119 int prio;
120
121 matchers = nat ? &fs_smfs->matchers_nat : &fs_smfs->matchers;
122 smfs_matcher = &matchers->smfs_matchers[ipv4 * 3 + tcp * 2 + gre];
123
124 if (refcount_inc_not_zero(&smfs_matcher->ref))
125 return smfs_matcher;
126
127 mutex_lock(&fs_smfs->lock);
128
129 /* Retry with lock, as another thread might have already created the relevant matcher
130 * till we acquired the lock
131 */
132 if (refcount_inc_not_zero(&smfs_matcher->ref))
133 goto out_unlock;
134
135 // Find next available priority in sorted used list
136 prio = 0;
137 prev = &matchers->used;
138 list_for_each_entry(m, &matchers->used, list) {
139 prev = &m->list;
140
141 if (m->prio == prio)
142 prio = m->prio + 1;
143 else
144 break;
145 }
146
147 tbl = nat ? fs_smfs->ct_nat_tbl : fs_smfs->ct_tbl;
148 dr_matcher = mlx5_ct_fs_smfs_matcher_create(fs, tbl, ipv4, tcp, gre, prio);
149 if (IS_ERR(dr_matcher)) {
150 netdev_warn(fs->netdev,
151 "ct_fs_smfs: failed to create matcher (nat %d, ipv4 %d, tcp %d, gre %d), err: %ld\n",
152 nat, ipv4, tcp, gre, PTR_ERR(dr_matcher));
153
154 smfs_matcher = ERR_CAST(dr_matcher);
155 goto out_unlock;
156 }
157
158 smfs_matcher->dr_matcher = dr_matcher;
159 smfs_matcher->prio = prio;
160 list_add(&smfs_matcher->list, prev);
161 refcount_set(&smfs_matcher->ref, 1);
162
163 out_unlock:
164 mutex_unlock(&fs_smfs->lock);
165 return smfs_matcher;
166 }
167
168 static void
mlx5_ct_fs_smfs_matcher_put(struct mlx5_ct_fs * fs,struct mlx5_ct_fs_smfs_matcher * smfs_matcher)169 mlx5_ct_fs_smfs_matcher_put(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_smfs_matcher *smfs_matcher)
170 {
171 struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
172
173 if (!refcount_dec_and_mutex_lock(&smfs_matcher->ref, &fs_smfs->lock))
174 return;
175
176 mlx5_smfs_matcher_destroy(smfs_matcher->dr_matcher);
177 list_del(&smfs_matcher->list);
178 mutex_unlock(&fs_smfs->lock);
179 }
180
181 static int
mlx5_ct_fs_smfs_init(struct mlx5_ct_fs * fs,struct mlx5_flow_table * ct,struct mlx5_flow_table * ct_nat,struct mlx5_flow_table * post_ct)182 mlx5_ct_fs_smfs_init(struct mlx5_ct_fs *fs, struct mlx5_flow_table *ct,
183 struct mlx5_flow_table *ct_nat, struct mlx5_flow_table *post_ct)
184 {
185 struct mlx5dr_table *ct_tbl, *ct_nat_tbl, *post_ct_tbl;
186 struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
187
188 post_ct_tbl = mlx5_smfs_table_get_from_fs_ft(post_ct);
189 ct_nat_tbl = mlx5_smfs_table_get_from_fs_ft(ct_nat);
190 ct_tbl = mlx5_smfs_table_get_from_fs_ft(ct);
191 fs_smfs->ct_nat = ct_nat;
192
193 if (!ct_tbl || !ct_nat_tbl || !post_ct_tbl) {
194 netdev_warn(fs->netdev, "ct_fs_smfs: failed to init, missing backing dr tables");
195 return -EOPNOTSUPP;
196 }
197
198 ct_dbg("using smfs steering");
199
200 fs_smfs->fwd_action = mlx5_smfs_action_create_dest_table(post_ct_tbl);
201 if (!fs_smfs->fwd_action) {
202 return -EINVAL;
203 }
204
205 fs_smfs->ct_tbl = ct_tbl;
206 fs_smfs->ct_nat_tbl = ct_nat_tbl;
207 mutex_init(&fs_smfs->lock);
208 INIT_LIST_HEAD(&fs_smfs->matchers.used);
209 INIT_LIST_HEAD(&fs_smfs->matchers_nat.used);
210
211 return 0;
212 }
213
214 static void
mlx5_ct_fs_smfs_destroy(struct mlx5_ct_fs * fs)215 mlx5_ct_fs_smfs_destroy(struct mlx5_ct_fs *fs)
216 {
217 struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
218
219 mlx5_smfs_action_destroy(fs_smfs->fwd_action);
220 }
221
222 static struct mlx5_ct_fs_rule *
mlx5_ct_fs_smfs_ct_rule_add(struct mlx5_ct_fs * fs,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr,struct flow_rule * flow_rule)223 mlx5_ct_fs_smfs_ct_rule_add(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec,
224 struct mlx5_flow_attr *attr, struct flow_rule *flow_rule)
225 {
226 struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
227 struct mlx5_ct_fs_smfs_matcher *smfs_matcher;
228 struct mlx5_ct_fs_smfs_rule *smfs_rule;
229 struct mlx5dr_action *actions[5];
230 struct mlx5dr_rule *rule;
231 int num_actions = 0, err;
232 bool nat, tcp, ipv4, gre;
233
234 if (!mlx5e_tc_ct_is_valid_flow_rule(fs->netdev, flow_rule))
235 return ERR_PTR(-EOPNOTSUPP);
236
237 smfs_rule = kzalloc(sizeof(*smfs_rule), GFP_KERNEL);
238 if (!smfs_rule)
239 return ERR_PTR(-ENOMEM);
240
241 smfs_rule->count_action = mlx5_smfs_action_create_flow_counter(mlx5_fc_id(attr->counter));
242 if (!smfs_rule->count_action) {
243 err = -EINVAL;
244 goto err_count;
245 }
246
247 actions[num_actions++] = smfs_rule->count_action;
248 actions[num_actions++] = attr->modify_hdr->fs_dr_action.dr_action;
249 actions[num_actions++] = fs_smfs->fwd_action;
250
251 nat = (attr->ft == fs_smfs->ct_nat);
252 ipv4 = mlx5e_tc_get_ip_version(spec, true) == 4;
253 tcp = MLX5_GET(fte_match_param, spec->match_value,
254 outer_headers.ip_protocol) == IPPROTO_TCP;
255 gre = MLX5_GET(fte_match_param, spec->match_value,
256 outer_headers.ip_protocol) == IPPROTO_GRE;
257
258 smfs_matcher = mlx5_ct_fs_smfs_matcher_get(fs, nat, ipv4, tcp, gre);
259 if (IS_ERR(smfs_matcher)) {
260 err = PTR_ERR(smfs_matcher);
261 goto err_matcher;
262 }
263
264 rule = mlx5_smfs_rule_create(smfs_matcher->dr_matcher, spec, num_actions, actions,
265 spec->flow_context.flow_source);
266 if (!rule) {
267 err = -EINVAL;
268 goto err_create;
269 }
270
271 smfs_rule->rule = rule;
272 smfs_rule->smfs_matcher = smfs_matcher;
273
274 return &smfs_rule->fs_rule;
275
276 err_create:
277 mlx5_ct_fs_smfs_matcher_put(fs, smfs_matcher);
278 err_matcher:
279 mlx5_smfs_action_destroy(smfs_rule->count_action);
280 err_count:
281 kfree(smfs_rule);
282 return ERR_PTR(err);
283 }
284
285 static void
mlx5_ct_fs_smfs_ct_rule_del(struct mlx5_ct_fs * fs,struct mlx5_ct_fs_rule * fs_rule)286 mlx5_ct_fs_smfs_ct_rule_del(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_rule *fs_rule)
287 {
288 struct mlx5_ct_fs_smfs_rule *smfs_rule = container_of(fs_rule,
289 struct mlx5_ct_fs_smfs_rule,
290 fs_rule);
291
292 mlx5_smfs_rule_destroy(smfs_rule->rule);
293 mlx5_ct_fs_smfs_matcher_put(fs, smfs_rule->smfs_matcher);
294 mlx5_smfs_action_destroy(smfs_rule->count_action);
295 kfree(smfs_rule);
296 }
297
mlx5_ct_fs_smfs_ct_rule_update(struct mlx5_ct_fs * fs,struct mlx5_ct_fs_rule * fs_rule,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)298 static int mlx5_ct_fs_smfs_ct_rule_update(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_rule *fs_rule,
299 struct mlx5_flow_spec *spec, struct mlx5_flow_attr *attr)
300 {
301 struct mlx5_ct_fs_smfs_rule *smfs_rule = container_of(fs_rule,
302 struct mlx5_ct_fs_smfs_rule,
303 fs_rule);
304 struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
305 struct mlx5dr_action *actions[3]; /* We only need to create 3 actions, see below. */
306 struct mlx5dr_rule *rule;
307
308 actions[0] = smfs_rule->count_action;
309 actions[1] = attr->modify_hdr->fs_dr_action.dr_action;
310 actions[2] = fs_smfs->fwd_action;
311
312 rule = mlx5_smfs_rule_create(smfs_rule->smfs_matcher->dr_matcher, spec,
313 ARRAY_SIZE(actions), actions, spec->flow_context.flow_source);
314 if (!rule)
315 return -EINVAL;
316
317 mlx5_smfs_rule_destroy(smfs_rule->rule);
318 smfs_rule->rule = rule;
319
320 return 0;
321 }
322
323 static struct mlx5_ct_fs_ops fs_smfs_ops = {
324 .ct_rule_add = mlx5_ct_fs_smfs_ct_rule_add,
325 .ct_rule_del = mlx5_ct_fs_smfs_ct_rule_del,
326 .ct_rule_update = mlx5_ct_fs_smfs_ct_rule_update,
327
328 .init = mlx5_ct_fs_smfs_init,
329 .destroy = mlx5_ct_fs_smfs_destroy,
330
331 .priv_size = sizeof(struct mlx5_ct_fs_smfs),
332 };
333
334 struct mlx5_ct_fs_ops *
mlx5_ct_fs_smfs_ops_get(void)335 mlx5_ct_fs_smfs_ops_get(void)
336 {
337 return &fs_smfs_ops;
338 }
339