1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2012 Chelsio Communications, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33
34 #ifdef TCP_OFFLOAD
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/module.h>
39 #include <sys/bus.h>
40 #include <sys/fnv_hash.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/rwlock.h>
44 #include <sys/socket.h>
45 #include <sys/sbuf.h>
46 #include <sys/taskqueue.h>
47 #include <net/if.h>
48 #include <net/if_types.h>
49 #include <net/ethernet.h>
50 #include <net/if_vlan_var.h>
51 #include <net/route.h>
52 #include <netinet/in.h>
53 #include <netinet/toecore.h>
54
55 #include "common/common.h"
56 #include "common/t4_msg.h"
57 #include "tom/t4_tom_l2t.h"
58 #include "tom/t4_tom.h"
59
60 #define VLAN_NONE 0xfff
61
62 static inline void
l2t_hold(struct l2t_data * d,struct l2t_entry * e)63 l2t_hold(struct l2t_data *d, struct l2t_entry *e)
64 {
65
66 if (atomic_fetchadd_int(&e->refcnt, 1) == 0) /* 0 -> 1 transition */
67 atomic_subtract_int(&d->nfree, 1);
68 }
69
70 static inline u_int
l2_hash(struct l2t_data * d,const struct sockaddr * sa,int ifindex)71 l2_hash(struct l2t_data *d, const struct sockaddr *sa, int ifindex)
72 {
73 u_int hash, half = d->l2t_size / 2, start = 0;
74 const void *key;
75 size_t len;
76
77 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
78 ("%s: sa %p has unexpected sa_family %d", __func__, sa,
79 sa->sa_family));
80
81 if (sa->sa_family == AF_INET) {
82 const struct sockaddr_in *sin = (const void *)sa;
83
84 key = &sin->sin_addr;
85 len = sizeof(sin->sin_addr);
86 } else {
87 const struct sockaddr_in6 *sin6 = (const void *)sa;
88
89 key = &sin6->sin6_addr;
90 len = sizeof(sin6->sin6_addr);
91 start = half;
92 }
93
94 hash = fnv_32_buf(key, len, FNV1_32_INIT);
95 hash = fnv_32_buf(&ifindex, sizeof(ifindex), hash);
96 hash %= half;
97
98 return (hash + start);
99 }
100
101 static inline int
l2_cmp(const struct sockaddr * sa,struct l2t_entry * e)102 l2_cmp(const struct sockaddr *sa, struct l2t_entry *e)
103 {
104
105 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
106 ("%s: sa %p has unexpected sa_family %d", __func__, sa,
107 sa->sa_family));
108
109 if (sa->sa_family == AF_INET) {
110 const struct sockaddr_in *sin = (const void *)sa;
111
112 return (e->addr[0] != sin->sin_addr.s_addr);
113 } else {
114 const struct sockaddr_in6 *sin6 = (const void *)sa;
115
116 return (memcmp(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr)));
117 }
118 }
119
120 static inline void
l2_store(const struct sockaddr * sa,struct l2t_entry * e)121 l2_store(const struct sockaddr *sa, struct l2t_entry *e)
122 {
123
124 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
125 ("%s: sa %p has unexpected sa_family %d", __func__, sa,
126 sa->sa_family));
127
128 if (sa->sa_family == AF_INET) {
129 const struct sockaddr_in *sin = (const void *)sa;
130
131 e->addr[0] = sin->sin_addr.s_addr;
132 e->ipv6 = 0;
133 } else {
134 const struct sockaddr_in6 *sin6 = (const void *)sa;
135
136 memcpy(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr));
137 e->ipv6 = 1;
138 }
139 }
140
141 /*
142 * Add a WR to an L2T entry's queue of work requests awaiting resolution.
143 * Must be called with the entry's lock held.
144 */
145 static inline void
arpq_enqueue(struct l2t_entry * e,struct wrqe * wr)146 arpq_enqueue(struct l2t_entry *e, struct wrqe *wr)
147 {
148 mtx_assert(&e->lock, MA_OWNED);
149
150 STAILQ_INSERT_TAIL(&e->wr_list, wr, link);
151 }
152
153 static inline void
send_pending(struct adapter * sc,struct l2t_entry * e)154 send_pending(struct adapter *sc, struct l2t_entry *e)
155 {
156 struct wrqe *wr;
157
158 mtx_assert(&e->lock, MA_OWNED);
159
160 while ((wr = STAILQ_FIRST(&e->wr_list)) != NULL) {
161 STAILQ_REMOVE_HEAD(&e->wr_list, link);
162 t4_wrq_tx(sc, wr);
163 }
164 }
165
166 static void
resolution_failed(struct adapter * sc,struct l2t_entry * e)167 resolution_failed(struct adapter *sc, struct l2t_entry *e)
168 {
169 struct tom_data *td = sc->tom_softc;
170
171 mtx_assert(&e->lock, MA_OWNED);
172
173 mtx_lock(&td->unsent_wr_lock);
174 STAILQ_CONCAT(&td->unsent_wr_list, &e->wr_list);
175 mtx_unlock(&td->unsent_wr_lock);
176
177 taskqueue_enqueue(taskqueue_thread, &td->reclaim_wr_resources);
178 }
179
180 static void
update_entry(struct adapter * sc,struct l2t_entry * e,uint8_t * lladdr,uint16_t vtag)181 update_entry(struct adapter *sc, struct l2t_entry *e, uint8_t *lladdr,
182 uint16_t vtag)
183 {
184
185 mtx_assert(&e->lock, MA_OWNED);
186
187 /*
188 * The entry may be in active use (e->refcount > 0) or not. We update
189 * it even when it's not as this simplifies the case where we decide to
190 * reuse the entry later.
191 */
192
193 if (lladdr == NULL &&
194 (e->state == L2T_STATE_RESOLVING || e->state == L2T_STATE_FAILED)) {
195 /*
196 * Never got a valid L2 address for this one. Just mark it as
197 * failed instead of removing it from the hash (for which we'd
198 * need to wlock the table).
199 */
200 e->state = L2T_STATE_FAILED;
201 resolution_failed(sc, e);
202 return;
203
204 } else if (lladdr == NULL) {
205
206 /* Valid or already-stale entry was deleted (or expired) */
207
208 KASSERT(e->state == L2T_STATE_VALID ||
209 e->state == L2T_STATE_STALE,
210 ("%s: lladdr NULL, state %d", __func__, e->state));
211
212 e->state = L2T_STATE_STALE;
213
214 } else {
215
216 if (e->state == L2T_STATE_RESOLVING ||
217 e->state == L2T_STATE_FAILED ||
218 memcmp(e->dmac, lladdr, ETHER_ADDR_LEN)) {
219
220 /* unresolved -> resolved; or dmac changed */
221
222 memcpy(e->dmac, lladdr, ETHER_ADDR_LEN);
223 e->vlan = vtag;
224 t4_write_l2e(e, 1);
225 }
226 e->state = L2T_STATE_VALID;
227 }
228 }
229
230 static int
resolve_entry(struct adapter * sc,struct l2t_entry * e)231 resolve_entry(struct adapter *sc, struct l2t_entry *e)
232 {
233 struct tom_data *td = sc->tom_softc;
234 struct toedev *tod = &td->tod;
235 struct sockaddr_in sin = {0};
236 struct sockaddr_in6 sin6 = {0};
237 struct sockaddr *sa;
238 uint8_t dmac[ETHER_HDR_LEN];
239 uint16_t vtag;
240 int rc;
241
242 if (e->ipv6 == 0) {
243 sin.sin_family = AF_INET;
244 sin.sin_len = sizeof(struct sockaddr_in);
245 sin.sin_addr.s_addr = e->addr[0];
246 sa = (void *)&sin;
247 } else {
248 sin6.sin6_family = AF_INET6;
249 sin6.sin6_len = sizeof(struct sockaddr_in6);
250 memcpy(&sin6.sin6_addr, &e->addr[0], sizeof(e->addr));
251 sa = (void *)&sin6;
252 }
253
254 vtag = EVL_MAKETAG(VLAN_NONE, 0, 0);
255 rc = toe_l2_resolve(tod, e->ifp, sa, dmac, &vtag);
256 if (rc == EWOULDBLOCK)
257 return (rc);
258
259 mtx_lock(&e->lock);
260 update_entry(sc, e, rc == 0 ? dmac : NULL, vtag);
261 mtx_unlock(&e->lock);
262
263 return (rc);
264 }
265
266 int
t4_l2t_send_slow(struct adapter * sc,struct wrqe * wr,struct l2t_entry * e)267 t4_l2t_send_slow(struct adapter *sc, struct wrqe *wr, struct l2t_entry *e)
268 {
269
270 again:
271 switch (e->state) {
272 case L2T_STATE_STALE: /* entry is stale, kick off revalidation */
273
274 if (resolve_entry(sc, e) != EWOULDBLOCK)
275 goto again; /* entry updated, re-examine state */
276
277 /* Fall through */
278
279 case L2T_STATE_VALID: /* fast-path, send the packet on */
280
281 t4_wrq_tx(sc, wr);
282 return (0);
283
284 case L2T_STATE_RESOLVING:
285 case L2T_STATE_SYNC_WRITE:
286
287 mtx_lock(&e->lock);
288 if (e->state != L2T_STATE_SYNC_WRITE &&
289 e->state != L2T_STATE_RESOLVING) {
290 /* state changed by the time we got here */
291 mtx_unlock(&e->lock);
292 goto again;
293 }
294 arpq_enqueue(e, wr);
295 mtx_unlock(&e->lock);
296
297 if (resolve_entry(sc, e) == EWOULDBLOCK)
298 break;
299
300 mtx_lock(&e->lock);
301 if (e->state == L2T_STATE_VALID && !STAILQ_EMPTY(&e->wr_list))
302 send_pending(sc, e);
303 if (e->state == L2T_STATE_FAILED)
304 resolution_failed(sc, e);
305 mtx_unlock(&e->lock);
306 break;
307
308 case L2T_STATE_FAILED:
309 return (EHOSTUNREACH);
310 }
311
312 return (0);
313 }
314
315 int
do_l2t_write_rpl2(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)316 do_l2t_write_rpl2(struct sge_iq *iq, const struct rss_header *rss,
317 struct mbuf *m)
318 {
319 struct adapter *sc = iq->adapter;
320 const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
321 unsigned int tid = GET_TID(rpl);
322 unsigned int idx = tid % L2T_SIZE;
323
324 if (__predict_false(rpl->status != CPL_ERR_NONE)) {
325 log(LOG_ERR,
326 "Unexpected L2T_WRITE_RPL (%u) for entry at hw_idx %u\n",
327 rpl->status, idx);
328 return (EINVAL);
329 }
330
331 if (tid & F_SYNC_WR) {
332 struct l2t_entry *e = &sc->l2t->l2tab[idx - sc->vres.l2t.start];
333
334 mtx_lock(&e->lock);
335 if (e->state != L2T_STATE_SWITCHING) {
336 send_pending(sc, e);
337 e->state = L2T_STATE_VALID;
338 }
339 mtx_unlock(&e->lock);
340 }
341
342 return (0);
343 }
344
345 /*
346 * The TOE wants an L2 table entry that it can use to reach the next hop over
347 * the specified port. Produce such an entry - create one if needed.
348 *
349 * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on
350 * top of the real cxgbe interface.
351 */
352 struct l2t_entry *
t4_l2t_get(struct port_info * pi,struct ifnet * ifp,struct sockaddr * sa)353 t4_l2t_get(struct port_info *pi, struct ifnet *ifp, struct sockaddr *sa)
354 {
355 struct l2t_entry *e;
356 struct adapter *sc = pi->adapter;
357 struct l2t_data *d = sc->l2t;
358 u_int hash, smt_idx = pi->port_id;
359 uint16_t vid, pcp, vtag;
360
361 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
362 ("%s: sa %p has unexpected sa_family %d", __func__, sa,
363 sa->sa_family));
364
365 vid = VLAN_NONE;
366 pcp = 0;
367 if (ifp->if_type == IFT_L2VLAN) {
368 VLAN_TAG(ifp, &vid);
369 VLAN_PCP(ifp, &pcp);
370 } else if (ifp->if_pcp != IFNET_PCP_NONE) {
371 vid = 0;
372 pcp = ifp->if_pcp;
373 }
374 vtag = EVL_MAKETAG(vid, pcp, 0);
375
376 hash = l2_hash(d, sa, ifp->if_index);
377 rw_wlock(&d->lock);
378 for (e = d->l2tab[hash].first; e; e = e->next) {
379 if (l2_cmp(sa, e) == 0 && e->ifp == ifp && e->vlan == vtag &&
380 e->smt_idx == smt_idx) {
381 l2t_hold(d, e);
382 goto done;
383 }
384 }
385
386 /* Need to allocate a new entry */
387 e = t4_alloc_l2e(d);
388 if (e) {
389 mtx_lock(&e->lock); /* avoid race with t4_l2t_free */
390 e->next = d->l2tab[hash].first;
391 d->l2tab[hash].first = e;
392
393 e->state = L2T_STATE_RESOLVING;
394 l2_store(sa, e);
395 e->ifp = ifp;
396 e->smt_idx = smt_idx;
397 e->hash = hash;
398 e->lport = pi->lport;
399 e->wrq = &sc->sge.ctrlq[pi->port_id];
400 e->iqid = sc->sge.ofld_rxq[pi->vi[0].first_ofld_rxq].iq.abs_id;
401 atomic_store_rel_int(&e->refcnt, 1);
402 e->vlan = vtag;
403 mtx_unlock(&e->lock);
404 }
405 done:
406 rw_wunlock(&d->lock);
407 return e;
408 }
409
410 /*
411 * Called when the host's ARP layer makes a change to some entry that is loaded
412 * into the HW L2 table.
413 */
414 void
t4_l2_update(struct toedev * tod,struct ifnet * ifp,struct sockaddr * sa,uint8_t * lladdr,uint16_t vtag)415 t4_l2_update(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa,
416 uint8_t *lladdr, uint16_t vtag)
417 {
418 struct adapter *sc = tod->tod_softc;
419 struct l2t_entry *e;
420 struct l2t_data *d = sc->l2t;
421 u_int hash;
422
423 KASSERT(d != NULL, ("%s: no L2 table", __func__));
424
425 hash = l2_hash(d, sa, ifp->if_index);
426 rw_rlock(&d->lock);
427 for (e = d->l2tab[hash].first; e; e = e->next) {
428 if (l2_cmp(sa, e) == 0 && e->ifp == ifp) {
429 mtx_lock(&e->lock);
430 if (atomic_load_acq_int(&e->refcnt))
431 goto found;
432 e->state = L2T_STATE_STALE;
433 mtx_unlock(&e->lock);
434 break;
435 }
436 }
437 rw_runlock(&d->lock);
438
439 /*
440 * This is of no interest to us. We've never had an offloaded
441 * connection to this destination, and we aren't attempting one right
442 * now.
443 */
444 return;
445
446 found:
447 rw_runlock(&d->lock);
448
449 KASSERT(e->state != L2T_STATE_UNUSED,
450 ("%s: unused entry in the hash.", __func__));
451
452 update_entry(sc, e, lladdr, vtag);
453 mtx_unlock(&e->lock);
454 }
455 #endif
456