1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
5 * All rights reserved.
6 * Written by: Navdeep Parhar <[email protected]>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include "opt_inet.h"
34 #include "opt_inet6.h"
35 #include "opt_ratelimit.h"
36
37 #ifdef TCP_OFFLOAD
38 #include <sys/param.h>
39 #include <sys/aio.h>
40 #include <sys/file.h>
41 #include <sys/kernel.h>
42 #include <sys/ktr.h>
43 #include <sys/module.h>
44 #include <sys/proc.h>
45 #include <sys/protosw.h>
46 #include <sys/domain.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
49 #include <sys/sglist.h>
50 #include <sys/taskqueue.h>
51 #include <netinet/in.h>
52 #include <netinet/in_pcb.h>
53 #include <netinet/ip.h>
54 #include <netinet/ip6.h>
55 #define TCPSTATES
56 #include <netinet/tcp_fsm.h>
57 #include <netinet/tcp_seq.h>
58 #include <netinet/tcp_var.h>
59 #include <netinet/toecore.h>
60
61 #include <security/mac/mac_framework.h>
62
63 #include <vm/vm.h>
64 #include <vm/vm_extern.h>
65 #include <vm/pmap.h>
66 #include <vm/vm_map.h>
67 #include <vm/vm_page.h>
68
69 #include "common/common.h"
70 #include "common/t4_msg.h"
71 #include "common/t4_regs.h"
72 #include "common/t4_tcb.h"
73 #include "tom/t4_tom_l2t.h"
74 #include "tom/t4_tom.h"
75
76 static void t4_aiotx_cancel(struct kaiocb *job);
77 static void t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep);
78
79 static size_t
aiotx_mbuf_pgoff(struct mbuf * m)80 aiotx_mbuf_pgoff(struct mbuf *m)
81 {
82 struct aiotx_buffer *ab;
83
84 MPASS(IS_AIOTX_MBUF(m));
85 ab = m->m_ext.ext_arg1;
86 return ((ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) % PAGE_SIZE);
87 }
88
89 static vm_page_t *
aiotx_mbuf_pages(struct mbuf * m)90 aiotx_mbuf_pages(struct mbuf *m)
91 {
92 struct aiotx_buffer *ab;
93 int npages;
94
95 MPASS(IS_AIOTX_MBUF(m));
96 ab = m->m_ext.ext_arg1;
97 npages = (ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) / PAGE_SIZE;
98 return (ab->ps.pages + npages);
99 }
100
101 void
send_flowc_wr(struct toepcb * toep,struct tcpcb * tp)102 send_flowc_wr(struct toepcb *toep, struct tcpcb *tp)
103 {
104 struct wrqe *wr;
105 struct fw_flowc_wr *flowc;
106 unsigned int nparams, flowclen, paramidx;
107 struct vi_info *vi = toep->vi;
108 struct port_info *pi = vi->pi;
109 struct adapter *sc = pi->adapter;
110 unsigned int pfvf = sc->pf << S_FW_VIID_PFN;
111 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
112
113 KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
114 ("%s: flowc for tid %u sent already", __func__, toep->tid));
115
116 if (tp != NULL)
117 nparams = 8;
118 else
119 nparams = 6;
120 if (ulp_mode(toep) == ULP_MODE_TLS)
121 nparams++;
122 if (toep->tls.fcplenmax != 0)
123 nparams++;
124 if (toep->params.tc_idx != -1) {
125 MPASS(toep->params.tc_idx >= 0 &&
126 toep->params.tc_idx < sc->chip_params->nsched_cls);
127 nparams++;
128 }
129
130 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
131
132 wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);
133 if (wr == NULL) {
134 /* XXX */
135 panic("%s: allocation failure.", __func__);
136 }
137 flowc = wrtod(wr);
138 memset(flowc, 0, wr->wr_len);
139
140 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
141 V_FW_FLOWC_WR_NPARAMS(nparams));
142 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
143 V_FW_WR_FLOWID(toep->tid));
144
145 #define FLOWC_PARAM(__m, __v) \
146 do { \
147 flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \
148 flowc->mnemval[paramidx].val = htobe32(__v); \
149 paramidx++; \
150 } while (0)
151
152 paramidx = 0;
153
154 FLOWC_PARAM(PFNVFN, pfvf);
155 FLOWC_PARAM(CH, pi->tx_chan);
156 FLOWC_PARAM(PORT, pi->tx_chan);
157 FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id);
158 FLOWC_PARAM(SNDBUF, toep->params.sndbuf);
159 FLOWC_PARAM(MSS, toep->params.emss);
160 if (tp) {
161 FLOWC_PARAM(SNDNXT, tp->snd_nxt);
162 FLOWC_PARAM(RCVNXT, tp->rcv_nxt);
163 }
164 CTR6(KTR_CXGBE,
165 "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x",
166 __func__, toep->tid, toep->params.emss, toep->params.sndbuf,
167 tp ? tp->snd_nxt : 0, tp ? tp->rcv_nxt : 0);
168
169 if (ulp_mode(toep) == ULP_MODE_TLS)
170 FLOWC_PARAM(ULP_MODE, ulp_mode(toep));
171 if (toep->tls.fcplenmax != 0)
172 FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax);
173 if (toep->params.tc_idx != -1)
174 FLOWC_PARAM(SCHEDCLASS, toep->params.tc_idx);
175 #undef FLOWC_PARAM
176
177 KASSERT(paramidx == nparams, ("nparams mismatch"));
178
179 txsd->tx_credits = howmany(flowclen, 16);
180 txsd->plen = 0;
181 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
182 ("%s: not enough credits (%d)", __func__, toep->tx_credits));
183 toep->tx_credits -= txsd->tx_credits;
184 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
185 toep->txsd_pidx = 0;
186 toep->txsd_avail--;
187
188 toep->flags |= TPF_FLOWC_WR_SENT;
189 t4_wrq_tx(sc, wr);
190 }
191
192 #ifdef RATELIMIT
193 /*
194 * Input is Bytes/second (so_max_pacing_rate), chip counts in Kilobits/second.
195 */
196 static int
update_tx_rate_limit(struct adapter * sc,struct toepcb * toep,u_int Bps)197 update_tx_rate_limit(struct adapter *sc, struct toepcb *toep, u_int Bps)
198 {
199 int tc_idx, rc;
200 const u_int kbps = (u_int) (uint64_t)Bps * 8ULL / 1000;
201 const int port_id = toep->vi->pi->port_id;
202
203 CTR3(KTR_CXGBE, "%s: tid %u, rate %uKbps", __func__, toep->tid, kbps);
204
205 if (kbps == 0) {
206 /* unbind */
207 tc_idx = -1;
208 } else {
209 rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx);
210 if (rc != 0)
211 return (rc);
212 MPASS(tc_idx >= 0 && tc_idx < sc->chip_params->nsched_cls);
213 }
214
215 if (toep->params.tc_idx != tc_idx) {
216 struct wrqe *wr;
217 struct fw_flowc_wr *flowc;
218 int nparams = 1, flowclen, flowclen16;
219 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
220
221 flowclen = sizeof(*flowc) + nparams * sizeof(struct
222 fw_flowc_mnemval);
223 flowclen16 = howmany(flowclen, 16);
224 if (toep->tx_credits < flowclen16 || toep->txsd_avail == 0 ||
225 (wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq)) == NULL) {
226 if (tc_idx >= 0)
227 t4_release_cl_rl(sc, port_id, tc_idx);
228 return (ENOMEM);
229 }
230
231 flowc = wrtod(wr);
232 memset(flowc, 0, wr->wr_len);
233
234 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
235 V_FW_FLOWC_WR_NPARAMS(nparams));
236 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(flowclen16) |
237 V_FW_WR_FLOWID(toep->tid));
238
239 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS;
240 if (tc_idx == -1)
241 flowc->mnemval[0].val = htobe32(0xff);
242 else
243 flowc->mnemval[0].val = htobe32(tc_idx);
244
245 txsd->tx_credits = flowclen16;
246 txsd->plen = 0;
247 toep->tx_credits -= txsd->tx_credits;
248 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
249 toep->txsd_pidx = 0;
250 toep->txsd_avail--;
251 t4_wrq_tx(sc, wr);
252 }
253
254 if (toep->params.tc_idx >= 0)
255 t4_release_cl_rl(sc, port_id, toep->params.tc_idx);
256 toep->params.tc_idx = tc_idx;
257
258 return (0);
259 }
260 #endif
261
262 void
send_reset(struct adapter * sc,struct toepcb * toep,uint32_t snd_nxt)263 send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
264 {
265 struct wrqe *wr;
266 struct cpl_abort_req *req;
267 int tid = toep->tid;
268 struct inpcb *inp = toep->inp;
269 struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */
270
271 INP_WLOCK_ASSERT(inp);
272
273 CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s",
274 __func__, toep->tid,
275 inp->inp_flags & INP_DROPPED ? "inp dropped" :
276 tcpstates[tp->t_state],
277 toep->flags, inp->inp_flags,
278 toep->flags & TPF_ABORT_SHUTDOWN ?
279 " (abort already in progress)" : "");
280
281 if (toep->flags & TPF_ABORT_SHUTDOWN)
282 return; /* abort already in progress */
283
284 toep->flags |= TPF_ABORT_SHUTDOWN;
285
286 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
287 ("%s: flowc_wr not sent for tid %d.", __func__, tid));
288
289 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
290 if (wr == NULL) {
291 /* XXX */
292 panic("%s: allocation failure.", __func__);
293 }
294 req = wrtod(wr);
295
296 INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid);
297 if (inp->inp_flags & INP_DROPPED)
298 req->rsvd0 = htobe32(snd_nxt);
299 else
300 req->rsvd0 = htobe32(tp->snd_nxt);
301 req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT);
302 req->cmd = CPL_ABORT_SEND_RST;
303
304 /*
305 * XXX: What's the correct way to tell that the inp hasn't been detached
306 * from its socket? Should I even be flushing the snd buffer here?
307 */
308 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
309 struct socket *so = inp->inp_socket;
310
311 if (so != NULL) /* because I'm not sure. See comment above */
312 sbflush(&so->so_snd);
313 }
314
315 t4_l2t_send(sc, wr, toep->l2te);
316 }
317
318 /*
319 * Called when a connection is established to translate the TCP options
320 * reported by HW to FreeBSD's native format.
321 */
322 static void
assign_rxopt(struct tcpcb * tp,uint16_t opt)323 assign_rxopt(struct tcpcb *tp, uint16_t opt)
324 {
325 struct toepcb *toep = tp->t_toe;
326 struct inpcb *inp = tp->t_inpcb;
327 struct adapter *sc = td_adapter(toep->td);
328
329 INP_LOCK_ASSERT(inp);
330
331 toep->params.mtu_idx = G_TCPOPT_MSS(opt);
332 tp->t_maxseg = sc->params.mtus[toep->params.mtu_idx];
333 if (inp->inp_inc.inc_flags & INC_ISIPV6)
334 tp->t_maxseg -= sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
335 else
336 tp->t_maxseg -= sizeof(struct ip) + sizeof(struct tcphdr);
337
338 toep->params.emss = tp->t_maxseg;
339 if (G_TCPOPT_TSTAMP(opt)) {
340 toep->params.tstamp = 1;
341 toep->params.emss -= TCPOLEN_TSTAMP_APPA;
342 tp->t_flags |= TF_RCVD_TSTMP; /* timestamps ok */
343 tp->ts_recent = 0; /* hmmm */
344 tp->ts_recent_age = tcp_ts_getticks();
345 } else
346 toep->params.tstamp = 0;
347
348 if (G_TCPOPT_SACK(opt)) {
349 toep->params.sack = 1;
350 tp->t_flags |= TF_SACK_PERMIT; /* should already be set */
351 } else {
352 toep->params.sack = 0;
353 tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */
354 }
355
356 if (G_TCPOPT_WSCALE_OK(opt))
357 tp->t_flags |= TF_RCVD_SCALE;
358
359 /* Doing window scaling? */
360 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
361 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
362 tp->rcv_scale = tp->request_r_scale;
363 tp->snd_scale = G_TCPOPT_SND_WSCALE(opt);
364 } else
365 toep->params.wscale = 0;
366
367 CTR6(KTR_CXGBE,
368 "assign_rxopt: tid %d, mtu_idx %u, emss %u, ts %u, sack %u, wscale %u",
369 toep->tid, toep->params.mtu_idx, toep->params.emss,
370 toep->params.tstamp, toep->params.sack, toep->params.wscale);
371 }
372
373 /*
374 * Completes some final bits of initialization for just established connections
375 * and changes their state to TCPS_ESTABLISHED.
376 *
377 * The ISNs are from the exchange of SYNs.
378 */
379 void
make_established(struct toepcb * toep,uint32_t iss,uint32_t irs,uint16_t opt)380 make_established(struct toepcb *toep, uint32_t iss, uint32_t irs, uint16_t opt)
381 {
382 struct inpcb *inp = toep->inp;
383 struct socket *so = inp->inp_socket;
384 struct tcpcb *tp = intotcpcb(inp);
385 uint16_t tcpopt = be16toh(opt);
386
387 INP_WLOCK_ASSERT(inp);
388 KASSERT(tp->t_state == TCPS_SYN_SENT ||
389 tp->t_state == TCPS_SYN_RECEIVED,
390 ("%s: TCP state %s", __func__, tcpstates[tp->t_state]));
391
392 CTR6(KTR_CXGBE, "%s: tid %d, so %p, inp %p, tp %p, toep %p",
393 __func__, toep->tid, so, inp, tp, toep);
394
395 tcp_state_change(tp, TCPS_ESTABLISHED);
396 tp->t_starttime = ticks;
397 TCPSTAT_INC(tcps_connects);
398
399 tp->irs = irs;
400 tcp_rcvseqinit(tp);
401 tp->rcv_wnd = (u_int)toep->params.opt0_bufsize << 10;
402 tp->rcv_adv += tp->rcv_wnd;
403 tp->last_ack_sent = tp->rcv_nxt;
404
405 tp->iss = iss;
406 tcp_sendseqinit(tp);
407 tp->snd_una = iss + 1;
408 tp->snd_nxt = iss + 1;
409 tp->snd_max = iss + 1;
410
411 assign_rxopt(tp, tcpopt);
412 send_flowc_wr(toep, tp);
413
414 soisconnected(so);
415 }
416
417 int
send_rx_credits(struct adapter * sc,struct toepcb * toep,int credits)418 send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
419 {
420 struct wrqe *wr;
421 struct cpl_rx_data_ack *req;
422 uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
423
424 KASSERT(credits >= 0, ("%s: %d credits", __func__, credits));
425
426 wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
427 if (wr == NULL)
428 return (0);
429 req = wrtod(wr);
430
431 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
432 req->credit_dack = htobe32(dack | V_RX_CREDITS(credits));
433
434 t4_wrq_tx(sc, wr);
435 return (credits);
436 }
437
438 void
send_rx_modulate(struct adapter * sc,struct toepcb * toep)439 send_rx_modulate(struct adapter *sc, struct toepcb *toep)
440 {
441 struct wrqe *wr;
442 struct cpl_rx_data_ack *req;
443
444 wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
445 if (wr == NULL)
446 return;
447 req = wrtod(wr);
448
449 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
450 req->credit_dack = htobe32(F_RX_MODULATE_RX);
451
452 t4_wrq_tx(sc, wr);
453 }
454
455 void
t4_rcvd_locked(struct toedev * tod,struct tcpcb * tp)456 t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp)
457 {
458 struct adapter *sc = tod->tod_softc;
459 struct inpcb *inp = tp->t_inpcb;
460 struct socket *so = inp->inp_socket;
461 struct sockbuf *sb = &so->so_rcv;
462 struct toepcb *toep = tp->t_toe;
463 int rx_credits;
464
465 INP_WLOCK_ASSERT(inp);
466 SOCKBUF_LOCK_ASSERT(sb);
467
468 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
469 if (ulp_mode(toep) == ULP_MODE_TLS) {
470 if (toep->tls.rcv_over >= rx_credits) {
471 toep->tls.rcv_over -= rx_credits;
472 rx_credits = 0;
473 } else {
474 rx_credits -= toep->tls.rcv_over;
475 toep->tls.rcv_over = 0;
476 }
477 }
478
479 if (rx_credits > 0 &&
480 (tp->rcv_wnd <= 32 * 1024 || rx_credits >= 64 * 1024 ||
481 (rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) ||
482 sbused(sb) + tp->rcv_wnd < sb->sb_lowat)) {
483 rx_credits = send_rx_credits(sc, toep, rx_credits);
484 tp->rcv_wnd += rx_credits;
485 tp->rcv_adv += rx_credits;
486 } else if (toep->flags & TPF_FORCE_CREDITS)
487 send_rx_modulate(sc, toep);
488 }
489
490 void
t4_rcvd(struct toedev * tod,struct tcpcb * tp)491 t4_rcvd(struct toedev *tod, struct tcpcb *tp)
492 {
493 struct inpcb *inp = tp->t_inpcb;
494 struct socket *so = inp->inp_socket;
495 struct sockbuf *sb = &so->so_rcv;
496
497 SOCKBUF_LOCK(sb);
498 t4_rcvd_locked(tod, tp);
499 SOCKBUF_UNLOCK(sb);
500 }
501
502 /*
503 * Close a connection by sending a CPL_CLOSE_CON_REQ message.
504 */
505 int
t4_close_conn(struct adapter * sc,struct toepcb * toep)506 t4_close_conn(struct adapter *sc, struct toepcb *toep)
507 {
508 struct wrqe *wr;
509 struct cpl_close_con_req *req;
510 unsigned int tid = toep->tid;
511
512 CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid,
513 toep->flags & TPF_FIN_SENT ? ", IGNORED" : "");
514
515 if (toep->flags & TPF_FIN_SENT)
516 return (0);
517
518 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
519 ("%s: flowc_wr not sent for tid %u.", __func__, tid));
520
521 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
522 if (wr == NULL) {
523 /* XXX */
524 panic("%s: allocation failure.", __func__);
525 }
526 req = wrtod(wr);
527
528 req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) |
529 V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr)));
530 req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) |
531 V_FW_WR_FLOWID(tid));
532 req->wr.wr_lo = cpu_to_be64(0);
533 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
534 req->rsvd = 0;
535
536 toep->flags |= TPF_FIN_SENT;
537 toep->flags &= ~TPF_SEND_FIN;
538 t4_l2t_send(sc, wr, toep->l2te);
539
540 return (0);
541 }
542
543 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
544 #define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16))
545
546 /* Maximum amount of immediate data we could stuff in a WR */
547 static inline int
max_imm_payload(int tx_credits)548 max_imm_payload(int tx_credits)
549 {
550 const int n = 1; /* Use no more than one desc for imm. data WR */
551
552 KASSERT(tx_credits >= 0 &&
553 tx_credits <= MAX_OFLD_TX_CREDITS,
554 ("%s: %d credits", __func__, tx_credits));
555
556 if (tx_credits < MIN_OFLD_TX_CREDITS)
557 return (0);
558
559 if (tx_credits >= (n * EQ_ESIZE) / 16)
560 return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr));
561 else
562 return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr));
563 }
564
565 /* Maximum number of SGL entries we could stuff in a WR */
566 static inline int
max_dsgl_nsegs(int tx_credits)567 max_dsgl_nsegs(int tx_credits)
568 {
569 int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */
570 int sge_pair_credits = tx_credits - MIN_OFLD_TX_CREDITS;
571
572 KASSERT(tx_credits >= 0 &&
573 tx_credits <= MAX_OFLD_TX_CREDITS,
574 ("%s: %d credits", __func__, tx_credits));
575
576 if (tx_credits < MIN_OFLD_TX_CREDITS)
577 return (0);
578
579 nseg += 2 * (sge_pair_credits * 16 / 24);
580 if ((sge_pair_credits * 16) % 24 == 16)
581 nseg++;
582
583 return (nseg);
584 }
585
586 static inline void
write_tx_wr(void * dst,struct toepcb * toep,unsigned int immdlen,unsigned int plen,uint8_t credits,int shove,int ulp_submode)587 write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
588 unsigned int plen, uint8_t credits, int shove, int ulp_submode)
589 {
590 struct fw_ofld_tx_data_wr *txwr = dst;
591
592 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
593 V_FW_WR_IMMDLEN(immdlen));
594 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
595 V_FW_WR_LEN16(credits));
596 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ulp_mode(toep)) |
597 V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove));
598 txwr->plen = htobe32(plen);
599
600 if (toep->params.tx_align > 0) {
601 if (plen < 2 * toep->params.emss)
602 txwr->lsodisable_to_flags |=
603 htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE);
604 else
605 txwr->lsodisable_to_flags |=
606 htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD |
607 (toep->params.nagle == 0 ? 0 :
608 F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE));
609 }
610 }
611
612 /*
613 * Generate a DSGL from a starting mbuf. The total number of segments and the
614 * maximum segments in any one mbuf are provided.
615 */
616 static void
write_tx_sgl(void * dst,struct mbuf * start,struct mbuf * stop,int nsegs,int n)617 write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n)
618 {
619 struct mbuf *m;
620 struct ulptx_sgl *usgl = dst;
621 int i, j, rc;
622 struct sglist sg;
623 struct sglist_seg segs[n];
624
625 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
626
627 sglist_init(&sg, n, segs);
628 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
629 V_ULPTX_NSGE(nsegs));
630
631 i = -1;
632 for (m = start; m != stop; m = m->m_next) {
633 if (IS_AIOTX_MBUF(m))
634 rc = sglist_append_vmpages(&sg, aiotx_mbuf_pages(m),
635 aiotx_mbuf_pgoff(m), m->m_len);
636 else
637 rc = sglist_append(&sg, mtod(m, void *), m->m_len);
638 if (__predict_false(rc != 0))
639 panic("%s: sglist_append %d", __func__, rc);
640
641 for (j = 0; j < sg.sg_nseg; i++, j++) {
642 if (i < 0) {
643 usgl->len0 = htobe32(segs[j].ss_len);
644 usgl->addr0 = htobe64(segs[j].ss_paddr);
645 } else {
646 usgl->sge[i / 2].len[i & 1] =
647 htobe32(segs[j].ss_len);
648 usgl->sge[i / 2].addr[i & 1] =
649 htobe64(segs[j].ss_paddr);
650 }
651 #ifdef INVARIANTS
652 nsegs--;
653 #endif
654 }
655 sglist_reset(&sg);
656 }
657 if (i & 1)
658 usgl->sge[i / 2].len[1] = htobe32(0);
659 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p",
660 __func__, nsegs, start, stop));
661 }
662
663 /*
664 * Max number of SGL entries an offload tx work request can have. This is 41
665 * (1 + 40) for a full 512B work request.
666 * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40)
667 */
668 #define OFLD_SGL_LEN (41)
669
670 /*
671 * Send data and/or a FIN to the peer.
672 *
673 * The socket's so_snd buffer consists of a stream of data starting with sb_mb
674 * and linked together with m_next. sb_sndptr, if set, is the last mbuf that
675 * was transmitted.
676 *
677 * drop indicates the number of bytes that should be dropped from the head of
678 * the send buffer. It is an optimization that lets do_fw4_ack avoid creating
679 * contention on the send buffer lock (before this change it used to do
680 * sowwakeup and then t4_push_frames right after that when recovering from tx
681 * stalls). When drop is set this function MUST drop the bytes and wake up any
682 * writers.
683 */
684 void
t4_push_frames(struct adapter * sc,struct toepcb * toep,int drop)685 t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
686 {
687 struct mbuf *sndptr, *m, *sb_sndptr;
688 struct fw_ofld_tx_data_wr *txwr;
689 struct wrqe *wr;
690 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
691 struct inpcb *inp = toep->inp;
692 struct tcpcb *tp = intotcpcb(inp);
693 struct socket *so = inp->inp_socket;
694 struct sockbuf *sb = &so->so_snd;
695 int tx_credits, shove, compl, sowwakeup;
696 struct ofld_tx_sdesc *txsd;
697 bool aiotx_mbuf_seen;
698
699 INP_WLOCK_ASSERT(inp);
700 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
701 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
702
703 KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
704 ulp_mode(toep) == ULP_MODE_TCPDDP ||
705 ulp_mode(toep) == ULP_MODE_TLS ||
706 ulp_mode(toep) == ULP_MODE_RDMA,
707 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
708
709 #ifdef VERBOSE_TRACES
710 CTR5(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
711 __func__, toep->tid, toep->flags, tp->t_flags, drop);
712 #endif
713 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
714 return;
715
716 #ifdef RATELIMIT
717 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
718 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
719 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
720 }
721 #endif
722
723 /*
724 * This function doesn't resume by itself. Someone else must clear the
725 * flag and call this function.
726 */
727 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
728 KASSERT(drop == 0,
729 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
730 return;
731 }
732
733 txsd = &toep->txsd[toep->txsd_pidx];
734 do {
735 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
736 max_imm = max_imm_payload(tx_credits);
737 max_nsegs = max_dsgl_nsegs(tx_credits);
738
739 SOCKBUF_LOCK(sb);
740 sowwakeup = drop;
741 if (drop) {
742 sbdrop_locked(sb, drop);
743 drop = 0;
744 }
745 sb_sndptr = sb->sb_sndptr;
746 sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb;
747 plen = 0;
748 nsegs = 0;
749 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
750 aiotx_mbuf_seen = false;
751 for (m = sndptr; m != NULL; m = m->m_next) {
752 int n;
753
754 if (IS_AIOTX_MBUF(m))
755 n = sglist_count_vmpages(aiotx_mbuf_pages(m),
756 aiotx_mbuf_pgoff(m), m->m_len);
757 else
758 n = sglist_count(mtod(m, void *), m->m_len);
759
760 nsegs += n;
761 plen += m->m_len;
762
763 /* This mbuf sent us _over_ the nsegs limit, back out */
764 if (plen > max_imm && nsegs > max_nsegs) {
765 nsegs -= n;
766 plen -= m->m_len;
767 if (plen == 0) {
768 /* Too few credits */
769 toep->flags |= TPF_TX_SUSPENDED;
770 if (sowwakeup) {
771 if (!TAILQ_EMPTY(
772 &toep->aiotx_jobq))
773 t4_aiotx_queue_toep(so,
774 toep);
775 sowwakeup_locked(so);
776 } else
777 SOCKBUF_UNLOCK(sb);
778 SOCKBUF_UNLOCK_ASSERT(sb);
779 return;
780 }
781 break;
782 }
783
784 if (IS_AIOTX_MBUF(m))
785 aiotx_mbuf_seen = true;
786 if (max_nsegs_1mbuf < n)
787 max_nsegs_1mbuf = n;
788 sb_sndptr = m; /* new sb->sb_sndptr if all goes well */
789
790 /* This mbuf put us right at the max_nsegs limit */
791 if (plen > max_imm && nsegs == max_nsegs) {
792 m = m->m_next;
793 break;
794 }
795 }
796
797 if (sbused(sb) > sb->sb_hiwat * 5 / 8 &&
798 toep->plen_nocompl + plen >= sb->sb_hiwat / 4)
799 compl = 1;
800 else
801 compl = 0;
802
803 if (sb->sb_flags & SB_AUTOSIZE &&
804 V_tcp_do_autosndbuf &&
805 sb->sb_hiwat < V_tcp_autosndbuf_max &&
806 sbused(sb) >= sb->sb_hiwat * 7 / 8) {
807 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
808 V_tcp_autosndbuf_max);
809
810 if (!sbreserve_locked(sb, newsize, so, NULL))
811 sb->sb_flags &= ~SB_AUTOSIZE;
812 else
813 sowwakeup = 1; /* room available */
814 }
815 if (sowwakeup) {
816 if (!TAILQ_EMPTY(&toep->aiotx_jobq))
817 t4_aiotx_queue_toep(so, toep);
818 sowwakeup_locked(so);
819 } else
820 SOCKBUF_UNLOCK(sb);
821 SOCKBUF_UNLOCK_ASSERT(sb);
822
823 /* nothing to send */
824 if (plen == 0) {
825 KASSERT(m == NULL,
826 ("%s: nothing to send, but m != NULL", __func__));
827 break;
828 }
829
830 if (__predict_false(toep->flags & TPF_FIN_SENT))
831 panic("%s: excess tx.", __func__);
832
833 shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
834 if (plen <= max_imm && !aiotx_mbuf_seen) {
835
836 /* Immediate data tx */
837
838 wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
839 toep->ofld_txq);
840 if (wr == NULL) {
841 /* XXX: how will we recover from this? */
842 toep->flags |= TPF_TX_SUSPENDED;
843 return;
844 }
845 txwr = wrtod(wr);
846 credits = howmany(wr->wr_len, 16);
847 write_tx_wr(txwr, toep, plen, plen, credits, shove, 0);
848 m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
849 nsegs = 0;
850 } else {
851 int wr_len;
852
853 /* DSGL tx */
854
855 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
856 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
857 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
858 if (wr == NULL) {
859 /* XXX: how will we recover from this? */
860 toep->flags |= TPF_TX_SUSPENDED;
861 return;
862 }
863 txwr = wrtod(wr);
864 credits = howmany(wr_len, 16);
865 write_tx_wr(txwr, toep, 0, plen, credits, shove, 0);
866 write_tx_sgl(txwr + 1, sndptr, m, nsegs,
867 max_nsegs_1mbuf);
868 if (wr_len & 0xf) {
869 uint64_t *pad = (uint64_t *)
870 ((uintptr_t)txwr + wr_len);
871 *pad = 0;
872 }
873 }
874
875 KASSERT(toep->tx_credits >= credits,
876 ("%s: not enough credits", __func__));
877
878 toep->tx_credits -= credits;
879 toep->tx_nocompl += credits;
880 toep->plen_nocompl += plen;
881 if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
882 toep->tx_nocompl >= toep->tx_total / 4)
883 compl = 1;
884
885 if (compl || ulp_mode(toep) == ULP_MODE_RDMA) {
886 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
887 toep->tx_nocompl = 0;
888 toep->plen_nocompl = 0;
889 }
890
891 tp->snd_nxt += plen;
892 tp->snd_max += plen;
893
894 SOCKBUF_LOCK(sb);
895 KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__));
896 sb->sb_sndptr = sb_sndptr;
897 SOCKBUF_UNLOCK(sb);
898
899 toep->flags |= TPF_TX_DATA_SENT;
900 if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
901 toep->flags |= TPF_TX_SUSPENDED;
902
903 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
904 txsd->plen = plen;
905 txsd->tx_credits = credits;
906 txsd++;
907 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
908 toep->txsd_pidx = 0;
909 txsd = &toep->txsd[0];
910 }
911 toep->txsd_avail--;
912
913 t4_l2t_send(sc, wr, toep->l2te);
914 } while (m != NULL);
915
916 /* Send a FIN if requested, but only if there's no more data to send */
917 if (m == NULL && toep->flags & TPF_SEND_FIN)
918 t4_close_conn(sc, toep);
919 }
920
921 static inline void
rqdrop_locked(struct mbufq * q,int plen)922 rqdrop_locked(struct mbufq *q, int plen)
923 {
924 struct mbuf *m;
925
926 while (plen > 0) {
927 m = mbufq_dequeue(q);
928
929 /* Too many credits. */
930 MPASS(m != NULL);
931 M_ASSERTPKTHDR(m);
932
933 /* Partial credits. */
934 MPASS(plen >= m->m_pkthdr.len);
935
936 plen -= m->m_pkthdr.len;
937 m_freem(m);
938 }
939 }
940
941 void
t4_push_pdus(struct adapter * sc,struct toepcb * toep,int drop)942 t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
943 {
944 struct mbuf *sndptr, *m;
945 struct fw_ofld_tx_data_wr *txwr;
946 struct wrqe *wr;
947 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
948 u_int adjusted_plen, ulp_submode;
949 struct inpcb *inp = toep->inp;
950 struct tcpcb *tp = intotcpcb(inp);
951 int tx_credits, shove;
952 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
953 struct mbufq *pduq = &toep->ulp_pduq;
954 static const u_int ulp_extra_len[] = {0, 4, 4, 8};
955
956 INP_WLOCK_ASSERT(inp);
957 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
958 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
959 KASSERT(ulp_mode(toep) == ULP_MODE_ISCSI,
960 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
961
962 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
963 return;
964
965 /*
966 * This function doesn't resume by itself. Someone else must clear the
967 * flag and call this function.
968 */
969 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
970 KASSERT(drop == 0,
971 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
972 return;
973 }
974
975 if (drop)
976 rqdrop_locked(&toep->ulp_pdu_reclaimq, drop);
977
978 while ((sndptr = mbufq_first(pduq)) != NULL) {
979 M_ASSERTPKTHDR(sndptr);
980
981 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
982 max_imm = max_imm_payload(tx_credits);
983 max_nsegs = max_dsgl_nsegs(tx_credits);
984
985 plen = 0;
986 nsegs = 0;
987 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
988 for (m = sndptr; m != NULL; m = m->m_next) {
989 int n = sglist_count(mtod(m, void *), m->m_len);
990
991 nsegs += n;
992 plen += m->m_len;
993
994 /*
995 * This mbuf would send us _over_ the nsegs limit.
996 * Suspend tx because the PDU can't be sent out.
997 */
998 if (plen > max_imm && nsegs > max_nsegs) {
999 toep->flags |= TPF_TX_SUSPENDED;
1000 return;
1001 }
1002
1003 if (max_nsegs_1mbuf < n)
1004 max_nsegs_1mbuf = n;
1005 }
1006
1007 if (__predict_false(toep->flags & TPF_FIN_SENT))
1008 panic("%s: excess tx.", __func__);
1009
1010 /*
1011 * We have a PDU to send. All of it goes out in one WR so 'm'
1012 * is NULL. A PDU's length is always a multiple of 4.
1013 */
1014 MPASS(m == NULL);
1015 MPASS((plen & 3) == 0);
1016 MPASS(sndptr->m_pkthdr.len == plen);
1017
1018 shove = !(tp->t_flags & TF_MORETOCOME);
1019 ulp_submode = mbuf_ulp_submode(sndptr);
1020 MPASS(ulp_submode < nitems(ulp_extra_len));
1021
1022 /*
1023 * plen doesn't include header and data digests, which are
1024 * generated and inserted in the right places by the TOE, but
1025 * they do occupy TCP sequence space and need to be accounted
1026 * for.
1027 */
1028 adjusted_plen = plen + ulp_extra_len[ulp_submode];
1029 if (plen <= max_imm) {
1030
1031 /* Immediate data tx */
1032
1033 wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
1034 toep->ofld_txq);
1035 if (wr == NULL) {
1036 /* XXX: how will we recover from this? */
1037 toep->flags |= TPF_TX_SUSPENDED;
1038 return;
1039 }
1040 txwr = wrtod(wr);
1041 credits = howmany(wr->wr_len, 16);
1042 write_tx_wr(txwr, toep, plen, adjusted_plen, credits,
1043 shove, ulp_submode);
1044 m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
1045 nsegs = 0;
1046 } else {
1047 int wr_len;
1048
1049 /* DSGL tx */
1050 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
1051 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
1052 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
1053 if (wr == NULL) {
1054 /* XXX: how will we recover from this? */
1055 toep->flags |= TPF_TX_SUSPENDED;
1056 return;
1057 }
1058 txwr = wrtod(wr);
1059 credits = howmany(wr_len, 16);
1060 write_tx_wr(txwr, toep, 0, adjusted_plen, credits,
1061 shove, ulp_submode);
1062 write_tx_sgl(txwr + 1, sndptr, m, nsegs,
1063 max_nsegs_1mbuf);
1064 if (wr_len & 0xf) {
1065 uint64_t *pad = (uint64_t *)
1066 ((uintptr_t)txwr + wr_len);
1067 *pad = 0;
1068 }
1069 }
1070
1071 KASSERT(toep->tx_credits >= credits,
1072 ("%s: not enough credits", __func__));
1073
1074 m = mbufq_dequeue(pduq);
1075 MPASS(m == sndptr);
1076 mbufq_enqueue(&toep->ulp_pdu_reclaimq, m);
1077
1078 toep->tx_credits -= credits;
1079 toep->tx_nocompl += credits;
1080 toep->plen_nocompl += plen;
1081 if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
1082 toep->tx_nocompl >= toep->tx_total / 4) {
1083 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
1084 toep->tx_nocompl = 0;
1085 toep->plen_nocompl = 0;
1086 }
1087
1088 tp->snd_nxt += adjusted_plen;
1089 tp->snd_max += adjusted_plen;
1090
1091 toep->flags |= TPF_TX_DATA_SENT;
1092 if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
1093 toep->flags |= TPF_TX_SUSPENDED;
1094
1095 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
1096 txsd->plen = plen;
1097 txsd->tx_credits = credits;
1098 txsd++;
1099 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
1100 toep->txsd_pidx = 0;
1101 txsd = &toep->txsd[0];
1102 }
1103 toep->txsd_avail--;
1104
1105 t4_l2t_send(sc, wr, toep->l2te);
1106 }
1107
1108 /* Send a FIN if requested, but only if there are no more PDUs to send */
1109 if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
1110 t4_close_conn(sc, toep);
1111 }
1112
1113 int
t4_tod_output(struct toedev * tod,struct tcpcb * tp)1114 t4_tod_output(struct toedev *tod, struct tcpcb *tp)
1115 {
1116 struct adapter *sc = tod->tod_softc;
1117 #ifdef INVARIANTS
1118 struct inpcb *inp = tp->t_inpcb;
1119 #endif
1120 struct toepcb *toep = tp->t_toe;
1121
1122 INP_WLOCK_ASSERT(inp);
1123 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1124 ("%s: inp %p dropped.", __func__, inp));
1125 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1126
1127 if (ulp_mode(toep) == ULP_MODE_ISCSI)
1128 t4_push_pdus(sc, toep, 0);
1129 else if (tls_tx_key(toep))
1130 t4_push_tls_records(sc, toep, 0);
1131 else
1132 t4_push_frames(sc, toep, 0);
1133
1134 return (0);
1135 }
1136
1137 int
t4_send_fin(struct toedev * tod,struct tcpcb * tp)1138 t4_send_fin(struct toedev *tod, struct tcpcb *tp)
1139 {
1140 struct adapter *sc = tod->tod_softc;
1141 #ifdef INVARIANTS
1142 struct inpcb *inp = tp->t_inpcb;
1143 #endif
1144 struct toepcb *toep = tp->t_toe;
1145
1146 INP_WLOCK_ASSERT(inp);
1147 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1148 ("%s: inp %p dropped.", __func__, inp));
1149 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1150
1151 toep->flags |= TPF_SEND_FIN;
1152 if (tp->t_state >= TCPS_ESTABLISHED) {
1153 if (ulp_mode(toep) == ULP_MODE_ISCSI)
1154 t4_push_pdus(sc, toep, 0);
1155 else if (tls_tx_key(toep))
1156 t4_push_tls_records(sc, toep, 0);
1157 else
1158 t4_push_frames(sc, toep, 0);
1159 }
1160
1161 return (0);
1162 }
1163
1164 int
t4_send_rst(struct toedev * tod,struct tcpcb * tp)1165 t4_send_rst(struct toedev *tod, struct tcpcb *tp)
1166 {
1167 struct adapter *sc = tod->tod_softc;
1168 #if defined(INVARIANTS)
1169 struct inpcb *inp = tp->t_inpcb;
1170 #endif
1171 struct toepcb *toep = tp->t_toe;
1172
1173 INP_WLOCK_ASSERT(inp);
1174 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1175 ("%s: inp %p dropped.", __func__, inp));
1176 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1177
1178 /* hmmmm */
1179 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
1180 ("%s: flowc for tid %u [%s] not sent already",
1181 __func__, toep->tid, tcpstates[tp->t_state]));
1182
1183 send_reset(sc, toep, 0);
1184 return (0);
1185 }
1186
1187 /*
1188 * Peer has sent us a FIN.
1189 */
1190 static int
do_peer_close(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1191 do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1192 {
1193 struct adapter *sc = iq->adapter;
1194 const struct cpl_peer_close *cpl = (const void *)(rss + 1);
1195 unsigned int tid = GET_TID(cpl);
1196 struct toepcb *toep = lookup_tid(sc, tid);
1197 struct inpcb *inp = toep->inp;
1198 struct tcpcb *tp = NULL;
1199 struct socket *so;
1200 struct epoch_tracker et;
1201 #ifdef INVARIANTS
1202 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1203 #endif
1204
1205 KASSERT(opcode == CPL_PEER_CLOSE,
1206 ("%s: unexpected opcode 0x%x", __func__, opcode));
1207 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1208
1209 if (__predict_false(toep->flags & TPF_SYNQE)) {
1210 /*
1211 * do_pass_establish must have run before do_peer_close and if
1212 * this is still a synqe instead of a toepcb then the connection
1213 * must be getting aborted.
1214 */
1215 MPASS(toep->flags & TPF_ABORT_SHUTDOWN);
1216 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
1217 toep, toep->flags);
1218 return (0);
1219 }
1220
1221 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1222
1223 CURVNET_SET(toep->vnet);
1224 INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1225 INP_WLOCK(inp);
1226 tp = intotcpcb(inp);
1227
1228 CTR6(KTR_CXGBE,
1229 "%s: tid %u (%s), toep_flags 0x%x, ddp_flags 0x%x, inp %p",
1230 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
1231 toep->ddp.flags, inp);
1232
1233 if (toep->flags & TPF_ABORT_SHUTDOWN)
1234 goto done;
1235
1236 tp->rcv_nxt++; /* FIN */
1237
1238 so = inp->inp_socket;
1239 socantrcvmore(so);
1240 if (ulp_mode(toep) == ULP_MODE_TCPDDP) {
1241 DDP_LOCK(toep);
1242 if (__predict_false(toep->ddp.flags &
1243 (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)))
1244 handle_ddp_close(toep, tp, cpl->rcv_nxt);
1245 DDP_UNLOCK(toep);
1246 }
1247
1248 if (ulp_mode(toep) != ULP_MODE_RDMA) {
1249 KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt),
1250 ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
1251 be32toh(cpl->rcv_nxt)));
1252 }
1253
1254 switch (tp->t_state) {
1255 case TCPS_SYN_RECEIVED:
1256 tp->t_starttime = ticks;
1257 /* FALLTHROUGH */
1258
1259 case TCPS_ESTABLISHED:
1260 tcp_state_change(tp, TCPS_CLOSE_WAIT);
1261 break;
1262
1263 case TCPS_FIN_WAIT_1:
1264 tcp_state_change(tp, TCPS_CLOSING);
1265 break;
1266
1267 case TCPS_FIN_WAIT_2:
1268 tcp_twstart(tp);
1269 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */
1270 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1271 CURVNET_RESTORE();
1272
1273 INP_WLOCK(inp);
1274 final_cpl_received(toep);
1275 return (0);
1276
1277 default:
1278 log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n",
1279 __func__, tid, tp->t_state);
1280 }
1281 done:
1282 INP_WUNLOCK(inp);
1283 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1284 CURVNET_RESTORE();
1285 return (0);
1286 }
1287
1288 /*
1289 * Peer has ACK'd our FIN.
1290 */
1291 static int
do_close_con_rpl(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1292 do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss,
1293 struct mbuf *m)
1294 {
1295 struct adapter *sc = iq->adapter;
1296 const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1);
1297 unsigned int tid = GET_TID(cpl);
1298 struct toepcb *toep = lookup_tid(sc, tid);
1299 struct inpcb *inp = toep->inp;
1300 struct tcpcb *tp = NULL;
1301 struct socket *so = NULL;
1302 struct epoch_tracker et;
1303 #ifdef INVARIANTS
1304 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1305 #endif
1306
1307 KASSERT(opcode == CPL_CLOSE_CON_RPL,
1308 ("%s: unexpected opcode 0x%x", __func__, opcode));
1309 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1310 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1311
1312 CURVNET_SET(toep->vnet);
1313 INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1314 INP_WLOCK(inp);
1315 tp = intotcpcb(inp);
1316
1317 CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x",
1318 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags);
1319
1320 if (toep->flags & TPF_ABORT_SHUTDOWN)
1321 goto done;
1322
1323 so = inp->inp_socket;
1324 tp->snd_una = be32toh(cpl->snd_nxt) - 1; /* exclude FIN */
1325
1326 switch (tp->t_state) {
1327 case TCPS_CLOSING: /* see TCPS_FIN_WAIT_2 in do_peer_close too */
1328 tcp_twstart(tp);
1329 release:
1330 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */
1331 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1332 CURVNET_RESTORE();
1333
1334 INP_WLOCK(inp);
1335 final_cpl_received(toep); /* no more CPLs expected */
1336
1337 return (0);
1338 case TCPS_LAST_ACK:
1339 if (tcp_close(tp))
1340 INP_WUNLOCK(inp);
1341 goto release;
1342
1343 case TCPS_FIN_WAIT_1:
1344 if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
1345 soisdisconnected(so);
1346 tcp_state_change(tp, TCPS_FIN_WAIT_2);
1347 break;
1348
1349 default:
1350 log(LOG_ERR,
1351 "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n",
1352 __func__, tid, tcpstates[tp->t_state]);
1353 }
1354 done:
1355 INP_WUNLOCK(inp);
1356 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1357 CURVNET_RESTORE();
1358 return (0);
1359 }
1360
1361 void
send_abort_rpl(struct adapter * sc,struct sge_wrq * ofld_txq,int tid,int rst_status)1362 send_abort_rpl(struct adapter *sc, struct sge_wrq *ofld_txq, int tid,
1363 int rst_status)
1364 {
1365 struct wrqe *wr;
1366 struct cpl_abort_rpl *cpl;
1367
1368 wr = alloc_wrqe(sizeof(*cpl), ofld_txq);
1369 if (wr == NULL) {
1370 /* XXX */
1371 panic("%s: allocation failure.", __func__);
1372 }
1373 cpl = wrtod(wr);
1374
1375 INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid);
1376 cpl->cmd = rst_status;
1377
1378 t4_wrq_tx(sc, wr);
1379 }
1380
1381 static int
abort_status_to_errno(struct tcpcb * tp,unsigned int abort_reason)1382 abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason)
1383 {
1384 switch (abort_reason) {
1385 case CPL_ERR_BAD_SYN:
1386 case CPL_ERR_CONN_RESET:
1387 return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET);
1388 case CPL_ERR_XMIT_TIMEDOUT:
1389 case CPL_ERR_PERSIST_TIMEDOUT:
1390 case CPL_ERR_FINWAIT2_TIMEDOUT:
1391 case CPL_ERR_KEEPALIVE_TIMEDOUT:
1392 return (ETIMEDOUT);
1393 default:
1394 return (EIO);
1395 }
1396 }
1397
1398 /*
1399 * TCP RST from the peer, timeout, or some other such critical error.
1400 */
1401 static int
do_abort_req(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1402 do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1403 {
1404 struct adapter *sc = iq->adapter;
1405 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
1406 unsigned int tid = GET_TID(cpl);
1407 struct toepcb *toep = lookup_tid(sc, tid);
1408 struct sge_wrq *ofld_txq = toep->ofld_txq;
1409 struct inpcb *inp;
1410 struct tcpcb *tp;
1411 struct epoch_tracker et;
1412 #ifdef INVARIANTS
1413 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1414 #endif
1415
1416 KASSERT(opcode == CPL_ABORT_REQ_RSS,
1417 ("%s: unexpected opcode 0x%x", __func__, opcode));
1418 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1419
1420 if (toep->flags & TPF_SYNQE)
1421 return (do_abort_req_synqe(iq, rss, m));
1422
1423 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1424
1425 if (negative_advice(cpl->status)) {
1426 CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)",
1427 __func__, cpl->status, tid, toep->flags);
1428 return (0); /* Ignore negative advice */
1429 }
1430
1431 inp = toep->inp;
1432 CURVNET_SET(toep->vnet);
1433 INP_INFO_RLOCK_ET(&V_tcbinfo, et); /* for tcp_close */
1434 INP_WLOCK(inp);
1435
1436 tp = intotcpcb(inp);
1437
1438 CTR6(KTR_CXGBE,
1439 "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d",
1440 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
1441 inp->inp_flags, cpl->status);
1442
1443 /*
1444 * If we'd initiated an abort earlier the reply to it is responsible for
1445 * cleaning up resources. Otherwise we tear everything down right here
1446 * right now. We owe the T4 a CPL_ABORT_RPL no matter what.
1447 */
1448 if (toep->flags & TPF_ABORT_SHUTDOWN) {
1449 INP_WUNLOCK(inp);
1450 goto done;
1451 }
1452 toep->flags |= TPF_ABORT_SHUTDOWN;
1453
1454 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
1455 struct socket *so = inp->inp_socket;
1456
1457 if (so != NULL)
1458 so_error_set(so, abort_status_to_errno(tp,
1459 cpl->status));
1460 tp = tcp_close(tp);
1461 if (tp == NULL)
1462 INP_WLOCK(inp); /* re-acquire */
1463 }
1464
1465 final_cpl_received(toep);
1466 done:
1467 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1468 CURVNET_RESTORE();
1469 send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST);
1470 return (0);
1471 }
1472
1473 /*
1474 * Reply to the CPL_ABORT_REQ (send_reset)
1475 */
1476 static int
do_abort_rpl(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1477 do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1478 {
1479 struct adapter *sc = iq->adapter;
1480 const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
1481 unsigned int tid = GET_TID(cpl);
1482 struct toepcb *toep = lookup_tid(sc, tid);
1483 struct inpcb *inp = toep->inp;
1484 #ifdef INVARIANTS
1485 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1486 #endif
1487
1488 KASSERT(opcode == CPL_ABORT_RPL_RSS,
1489 ("%s: unexpected opcode 0x%x", __func__, opcode));
1490 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1491
1492 if (toep->flags & TPF_SYNQE)
1493 return (do_abort_rpl_synqe(iq, rss, m));
1494
1495 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1496
1497 CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d",
1498 __func__, tid, toep, inp, cpl->status);
1499
1500 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1501 ("%s: wasn't expecting abort reply", __func__));
1502
1503 INP_WLOCK(inp);
1504 final_cpl_received(toep);
1505
1506 return (0);
1507 }
1508
1509 static int
do_rx_data(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1510 do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1511 {
1512 struct adapter *sc = iq->adapter;
1513 const struct cpl_rx_data *cpl = mtod(m, const void *);
1514 unsigned int tid = GET_TID(cpl);
1515 struct toepcb *toep = lookup_tid(sc, tid);
1516 struct inpcb *inp = toep->inp;
1517 struct tcpcb *tp;
1518 struct socket *so;
1519 struct sockbuf *sb;
1520 struct epoch_tracker et;
1521 int len, rx_credits;
1522 uint32_t ddp_placed = 0;
1523
1524 if (__predict_false(toep->flags & TPF_SYNQE)) {
1525 /*
1526 * do_pass_establish must have run before do_rx_data and if this
1527 * is still a synqe instead of a toepcb then the connection must
1528 * be getting aborted.
1529 */
1530 MPASS(toep->flags & TPF_ABORT_SHUTDOWN);
1531 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
1532 toep, toep->flags);
1533 m_freem(m);
1534 return (0);
1535 }
1536
1537 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1538
1539 /* strip off CPL header */
1540 m_adj(m, sizeof(*cpl));
1541 len = m->m_pkthdr.len;
1542
1543 INP_WLOCK(inp);
1544 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
1545 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
1546 __func__, tid, len, inp->inp_flags);
1547 INP_WUNLOCK(inp);
1548 m_freem(m);
1549 return (0);
1550 }
1551
1552 tp = intotcpcb(inp);
1553
1554 if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq)))
1555 ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt;
1556
1557 tp->rcv_nxt += len;
1558 if (tp->rcv_wnd < len) {
1559 KASSERT(ulp_mode(toep) == ULP_MODE_RDMA,
1560 ("%s: negative window size", __func__));
1561 }
1562
1563 tp->rcv_wnd -= len;
1564 tp->t_rcvtime = ticks;
1565
1566 if (ulp_mode(toep) == ULP_MODE_TCPDDP)
1567 DDP_LOCK(toep);
1568 so = inp_inpcbtosocket(inp);
1569 sb = &so->so_rcv;
1570 SOCKBUF_LOCK(sb);
1571
1572 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
1573 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
1574 __func__, tid, len);
1575 m_freem(m);
1576 SOCKBUF_UNLOCK(sb);
1577 if (ulp_mode(toep) == ULP_MODE_TCPDDP)
1578 DDP_UNLOCK(toep);
1579 INP_WUNLOCK(inp);
1580
1581 CURVNET_SET(toep->vnet);
1582 INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1583 INP_WLOCK(inp);
1584 tp = tcp_drop(tp, ECONNRESET);
1585 if (tp)
1586 INP_WUNLOCK(inp);
1587 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1588 CURVNET_RESTORE();
1589
1590 return (0);
1591 }
1592
1593 /* receive buffer autosize */
1594 MPASS(toep->vnet == so->so_vnet);
1595 CURVNET_SET(toep->vnet);
1596 if (sb->sb_flags & SB_AUTOSIZE &&
1597 V_tcp_do_autorcvbuf &&
1598 sb->sb_hiwat < V_tcp_autorcvbuf_max &&
1599 len > (sbspace(sb) / 8 * 7)) {
1600 unsigned int hiwat = sb->sb_hiwat;
1601 unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc,
1602 V_tcp_autorcvbuf_max);
1603
1604 if (!sbreserve_locked(sb, newsize, so, NULL))
1605 sb->sb_flags &= ~SB_AUTOSIZE;
1606 }
1607
1608 if (ulp_mode(toep) == ULP_MODE_TCPDDP) {
1609 int changed = !(toep->ddp.flags & DDP_ON) ^ cpl->ddp_off;
1610
1611 if (toep->ddp.waiting_count != 0 || toep->ddp.active_count != 0)
1612 CTR3(KTR_CXGBE, "%s: tid %u, non-ddp rx (%d bytes)",
1613 __func__, tid, len);
1614
1615 if (changed) {
1616 if (toep->ddp.flags & DDP_SC_REQ)
1617 toep->ddp.flags ^= DDP_ON | DDP_SC_REQ;
1618 else {
1619 KASSERT(cpl->ddp_off == 1,
1620 ("%s: DDP switched on by itself.",
1621 __func__));
1622
1623 /* Fell out of DDP mode */
1624 toep->ddp.flags &= ~DDP_ON;
1625 CTR1(KTR_CXGBE, "%s: fell out of DDP mode",
1626 __func__);
1627
1628 insert_ddp_data(toep, ddp_placed);
1629 }
1630 }
1631
1632 if (toep->ddp.flags & DDP_ON) {
1633 /*
1634 * CPL_RX_DATA with DDP on can only be an indicate.
1635 * Start posting queued AIO requests via DDP. The
1636 * payload that arrived in this indicate is appended
1637 * to the socket buffer as usual.
1638 */
1639 handle_ddp_indicate(toep);
1640 }
1641 }
1642
1643 sbappendstream_locked(sb, m, 0);
1644 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
1645 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) {
1646 rx_credits = send_rx_credits(sc, toep, rx_credits);
1647 tp->rcv_wnd += rx_credits;
1648 tp->rcv_adv += rx_credits;
1649 }
1650
1651 if (ulp_mode(toep) == ULP_MODE_TCPDDP && toep->ddp.waiting_count > 0 &&
1652 sbavail(sb) != 0) {
1653 CTR2(KTR_CXGBE, "%s: tid %u queueing AIO task", __func__,
1654 tid);
1655 ddp_queue_toep(toep);
1656 }
1657 sorwakeup_locked(so);
1658 SOCKBUF_UNLOCK_ASSERT(sb);
1659 if (ulp_mode(toep) == ULP_MODE_TCPDDP)
1660 DDP_UNLOCK(toep);
1661
1662 INP_WUNLOCK(inp);
1663 CURVNET_RESTORE();
1664 return (0);
1665 }
1666
1667 static int
do_fw4_ack(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1668 do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1669 {
1670 struct adapter *sc = iq->adapter;
1671 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
1672 unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
1673 struct toepcb *toep = lookup_tid(sc, tid);
1674 struct inpcb *inp;
1675 struct tcpcb *tp;
1676 struct socket *so;
1677 uint8_t credits = cpl->credits;
1678 struct ofld_tx_sdesc *txsd;
1679 int plen;
1680 #ifdef INVARIANTS
1681 unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl)));
1682 #endif
1683
1684 /*
1685 * Very unusual case: we'd sent a flowc + abort_req for a synq entry and
1686 * now this comes back carrying the credits for the flowc.
1687 */
1688 if (__predict_false(toep->flags & TPF_SYNQE)) {
1689 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1690 ("%s: credits for a synq entry %p", __func__, toep));
1691 return (0);
1692 }
1693
1694 inp = toep->inp;
1695
1696 KASSERT(opcode == CPL_FW4_ACK,
1697 ("%s: unexpected opcode 0x%x", __func__, opcode));
1698 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1699 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1700
1701 INP_WLOCK(inp);
1702
1703 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) {
1704 INP_WUNLOCK(inp);
1705 return (0);
1706 }
1707
1708 KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0,
1709 ("%s: inp_flags 0x%x", __func__, inp->inp_flags));
1710
1711 tp = intotcpcb(inp);
1712
1713 if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) {
1714 tcp_seq snd_una = be32toh(cpl->snd_una);
1715
1716 #ifdef INVARIANTS
1717 if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) {
1718 log(LOG_ERR,
1719 "%s: unexpected seq# %x for TID %u, snd_una %x\n",
1720 __func__, snd_una, toep->tid, tp->snd_una);
1721 }
1722 #endif
1723
1724 if (tp->snd_una != snd_una) {
1725 tp->snd_una = snd_una;
1726 tp->ts_recent_age = tcp_ts_getticks();
1727 }
1728 }
1729
1730 #ifdef VERBOSE_TRACES
1731 CTR3(KTR_CXGBE, "%s: tid %d credits %u", __func__, tid, credits);
1732 #endif
1733 so = inp->inp_socket;
1734 txsd = &toep->txsd[toep->txsd_cidx];
1735 plen = 0;
1736 while (credits) {
1737 KASSERT(credits >= txsd->tx_credits,
1738 ("%s: too many (or partial) credits", __func__));
1739 credits -= txsd->tx_credits;
1740 toep->tx_credits += txsd->tx_credits;
1741 plen += txsd->plen;
1742 if (txsd->iv_buffer) {
1743 free(txsd->iv_buffer, M_CXGBE);
1744 txsd->iv_buffer = NULL;
1745 }
1746 txsd++;
1747 toep->txsd_avail++;
1748 KASSERT(toep->txsd_avail <= toep->txsd_total,
1749 ("%s: txsd avail > total", __func__));
1750 if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) {
1751 txsd = &toep->txsd[0];
1752 toep->txsd_cidx = 0;
1753 }
1754 }
1755
1756 if (toep->tx_credits == toep->tx_total) {
1757 toep->tx_nocompl = 0;
1758 toep->plen_nocompl = 0;
1759 }
1760
1761 if (toep->flags & TPF_TX_SUSPENDED &&
1762 toep->tx_credits >= toep->tx_total / 4) {
1763 #ifdef VERBOSE_TRACES
1764 CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__,
1765 tid);
1766 #endif
1767 toep->flags &= ~TPF_TX_SUSPENDED;
1768 CURVNET_SET(toep->vnet);
1769 if (ulp_mode(toep) == ULP_MODE_ISCSI)
1770 t4_push_pdus(sc, toep, plen);
1771 else if (tls_tx_key(toep))
1772 t4_push_tls_records(sc, toep, plen);
1773 else
1774 t4_push_frames(sc, toep, plen);
1775 CURVNET_RESTORE();
1776 } else if (plen > 0) {
1777 struct sockbuf *sb = &so->so_snd;
1778 int sbu;
1779
1780 SOCKBUF_LOCK(sb);
1781 sbu = sbused(sb);
1782 if (ulp_mode(toep) == ULP_MODE_ISCSI) {
1783
1784 if (__predict_false(sbu > 0)) {
1785 /*
1786 * The data trasmitted before the tid's ULP mode
1787 * changed to ISCSI is still in so_snd.
1788 * Incoming credits should account for so_snd
1789 * first.
1790 */
1791 sbdrop_locked(sb, min(sbu, plen));
1792 plen -= min(sbu, plen);
1793 }
1794 sowwakeup_locked(so); /* unlocks so_snd */
1795 rqdrop_locked(&toep->ulp_pdu_reclaimq, plen);
1796 } else {
1797 #ifdef VERBOSE_TRACES
1798 CTR3(KTR_CXGBE, "%s: tid %d dropped %d bytes", __func__,
1799 tid, plen);
1800 #endif
1801 sbdrop_locked(sb, plen);
1802 if (tls_tx_key(toep)) {
1803 struct tls_ofld_info *tls_ofld = &toep->tls;
1804
1805 MPASS(tls_ofld->sb_off >= plen);
1806 tls_ofld->sb_off -= plen;
1807 }
1808 if (!TAILQ_EMPTY(&toep->aiotx_jobq))
1809 t4_aiotx_queue_toep(so, toep);
1810 sowwakeup_locked(so); /* unlocks so_snd */
1811 }
1812 SOCKBUF_UNLOCK_ASSERT(sb);
1813 }
1814
1815 INP_WUNLOCK(inp);
1816
1817 return (0);
1818 }
1819
1820 void
t4_set_tcb_field(struct adapter * sc,struct sge_wrq * wrq,struct toepcb * toep,uint16_t word,uint64_t mask,uint64_t val,int reply,int cookie)1821 t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep,
1822 uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie)
1823 {
1824 struct wrqe *wr;
1825 struct cpl_set_tcb_field *req;
1826 struct ofld_tx_sdesc *txsd;
1827
1828 MPASS((cookie & ~M_COOKIE) == 0);
1829 if (reply) {
1830 MPASS(cookie != CPL_COOKIE_RESERVED);
1831 }
1832
1833 wr = alloc_wrqe(sizeof(*req), wrq);
1834 if (wr == NULL) {
1835 /* XXX */
1836 panic("%s: allocation failure.", __func__);
1837 }
1838 req = wrtod(wr);
1839
1840 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid);
1841 req->reply_ctrl = htobe16(V_QUEUENO(toep->ofld_rxq->iq.abs_id));
1842 if (reply == 0)
1843 req->reply_ctrl |= htobe16(F_NO_REPLY);
1844 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie));
1845 req->mask = htobe64(mask);
1846 req->val = htobe64(val);
1847 if ((wrq->eq.flags & EQ_TYPEMASK) == EQ_OFLD) {
1848 txsd = &toep->txsd[toep->txsd_pidx];
1849 txsd->tx_credits = howmany(sizeof(*req), 16);
1850 txsd->plen = 0;
1851 KASSERT(toep->tx_credits >= txsd->tx_credits &&
1852 toep->txsd_avail > 0,
1853 ("%s: not enough credits (%d)", __func__,
1854 toep->tx_credits));
1855 toep->tx_credits -= txsd->tx_credits;
1856 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
1857 toep->txsd_pidx = 0;
1858 toep->txsd_avail--;
1859 }
1860
1861 t4_wrq_tx(sc, wr);
1862 }
1863
1864 void
t4_init_cpl_io_handlers(void)1865 t4_init_cpl_io_handlers(void)
1866 {
1867
1868 t4_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close);
1869 t4_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl);
1870 t4_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req);
1871 t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl,
1872 CPL_COOKIE_TOM);
1873 t4_register_cpl_handler(CPL_RX_DATA, do_rx_data);
1874 t4_register_shared_cpl_handler(CPL_FW4_ACK, do_fw4_ack, CPL_COOKIE_TOM);
1875 }
1876
1877 void
t4_uninit_cpl_io_handlers(void)1878 t4_uninit_cpl_io_handlers(void)
1879 {
1880
1881 t4_register_cpl_handler(CPL_PEER_CLOSE, NULL);
1882 t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL);
1883 t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL);
1884 t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, NULL, CPL_COOKIE_TOM);
1885 t4_register_cpl_handler(CPL_RX_DATA, NULL);
1886 t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM);
1887 }
1888
1889 /*
1890 * Use the 'backend3' field in AIO jobs to store the amount of data
1891 * sent by the AIO job so far and the 'backend4' field to hold an
1892 * error that should be reported when the job is completed.
1893 */
1894 #define aio_sent backend3
1895 #define aio_error backend4
1896
1897 #define jobtotid(job) \
1898 (((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid)
1899
1900 static void
free_aiotx_buffer(struct aiotx_buffer * ab)1901 free_aiotx_buffer(struct aiotx_buffer *ab)
1902 {
1903 struct kaiocb *job;
1904 long status;
1905 int error;
1906
1907 if (refcount_release(&ab->refcount) == 0)
1908 return;
1909
1910 job = ab->job;
1911 error = job->aio_error;
1912 status = job->aio_sent;
1913 vm_page_unhold_pages(ab->ps.pages, ab->ps.npages);
1914 free(ab, M_CXGBE);
1915 #ifdef VERBOSE_TRACES
1916 CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__,
1917 jobtotid(job), job, status, error);
1918 #endif
1919 if (error == ECANCELED && status != 0)
1920 error = 0;
1921 if (error == ECANCELED)
1922 aio_cancel(job);
1923 else if (error)
1924 aio_complete(job, -1, error);
1925 else
1926 aio_complete(job, status, 0);
1927 }
1928
1929 static void
t4_aiotx_mbuf_free(struct mbuf * m)1930 t4_aiotx_mbuf_free(struct mbuf *m)
1931 {
1932 struct aiotx_buffer *ab = m->m_ext.ext_arg1;
1933
1934 #ifdef VERBOSE_TRACES
1935 CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__,
1936 m->m_len, jobtotid(ab->job));
1937 #endif
1938 free_aiotx_buffer(ab);
1939 }
1940
1941 /*
1942 * Hold the buffer backing an AIO request and return an AIO transmit
1943 * buffer.
1944 */
1945 static int
hold_aio(struct kaiocb * job)1946 hold_aio(struct kaiocb *job)
1947 {
1948 struct aiotx_buffer *ab;
1949 struct vmspace *vm;
1950 vm_map_t map;
1951 vm_offset_t start, end, pgoff;
1952 int n;
1953
1954 MPASS(job->backend1 == NULL);
1955
1956 /*
1957 * The AIO subsystem will cancel and drain all requests before
1958 * permitting a process to exit or exec, so p_vmspace should
1959 * be stable here.
1960 */
1961 vm = job->userproc->p_vmspace;
1962 map = &vm->vm_map;
1963 start = (uintptr_t)job->uaiocb.aio_buf;
1964 pgoff = start & PAGE_MASK;
1965 end = round_page(start + job->uaiocb.aio_nbytes);
1966 start = trunc_page(start);
1967 n = atop(end - start);
1968
1969 ab = malloc(sizeof(*ab) + n * sizeof(vm_page_t), M_CXGBE, M_WAITOK |
1970 M_ZERO);
1971 refcount_init(&ab->refcount, 1);
1972 ab->ps.pages = (vm_page_t *)(ab + 1);
1973 ab->ps.npages = vm_fault_quick_hold_pages(map, start, end - start,
1974 VM_PROT_WRITE, ab->ps.pages, n);
1975 if (ab->ps.npages < 0) {
1976 free(ab, M_CXGBE);
1977 return (EFAULT);
1978 }
1979
1980 KASSERT(ab->ps.npages == n,
1981 ("hold_aio: page count mismatch: %d vs %d", ab->ps.npages, n));
1982
1983 ab->ps.offset = pgoff;
1984 ab->ps.len = job->uaiocb.aio_nbytes;
1985 ab->job = job;
1986 job->backend1 = ab;
1987 #ifdef VERBOSE_TRACES
1988 CTR5(KTR_CXGBE, "%s: tid %d, new pageset %p for job %p, npages %d",
1989 __func__, jobtotid(job), &ab->ps, job, ab->ps.npages);
1990 #endif
1991 return (0);
1992 }
1993
1994 static void
t4_aiotx_process_job(struct toepcb * toep,struct socket * so,struct kaiocb * job)1995 t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job)
1996 {
1997 struct sockbuf *sb;
1998 struct file *fp;
1999 struct aiotx_buffer *ab;
2000 struct inpcb *inp;
2001 struct tcpcb *tp;
2002 struct mbuf *m;
2003 int error;
2004 bool moretocome, sendmore;
2005
2006 sb = &so->so_snd;
2007 SOCKBUF_UNLOCK(sb);
2008 fp = job->fd_file;
2009 ab = job->backend1;
2010 m = NULL;
2011
2012 #ifdef MAC
2013 error = mac_socket_check_send(fp->f_cred, so);
2014 if (error != 0)
2015 goto out;
2016 #endif
2017
2018 if (ab == NULL) {
2019 error = hold_aio(job);
2020 if (error != 0)
2021 goto out;
2022 ab = job->backend1;
2023 }
2024
2025 /* Inline sosend_generic(). */
2026
2027 job->msgsnd = 1;
2028
2029 error = sblock(sb, SBL_WAIT);
2030 MPASS(error == 0);
2031
2032 sendanother:
2033 m = m_get(M_WAITOK, MT_DATA);
2034
2035 SOCKBUF_LOCK(sb);
2036 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
2037 SOCKBUF_UNLOCK(sb);
2038 sbunlock(sb);
2039 if ((so->so_options & SO_NOSIGPIPE) == 0) {
2040 PROC_LOCK(job->userproc);
2041 kern_psignal(job->userproc, SIGPIPE);
2042 PROC_UNLOCK(job->userproc);
2043 }
2044 error = EPIPE;
2045 goto out;
2046 }
2047 if (so->so_error) {
2048 error = so->so_error;
2049 so->so_error = 0;
2050 SOCKBUF_UNLOCK(sb);
2051 sbunlock(sb);
2052 goto out;
2053 }
2054 if ((so->so_state & SS_ISCONNECTED) == 0) {
2055 SOCKBUF_UNLOCK(sb);
2056 sbunlock(sb);
2057 error = ENOTCONN;
2058 goto out;
2059 }
2060 if (sbspace(sb) < sb->sb_lowat) {
2061 MPASS(job->aio_sent == 0 || !(so->so_state & SS_NBIO));
2062
2063 /*
2064 * Don't block if there is too little room in the socket
2065 * buffer. Instead, requeue the request.
2066 */
2067 if (!aio_set_cancel_function(job, t4_aiotx_cancel)) {
2068 SOCKBUF_UNLOCK(sb);
2069 sbunlock(sb);
2070 error = ECANCELED;
2071 goto out;
2072 }
2073 TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list);
2074 SOCKBUF_UNLOCK(sb);
2075 sbunlock(sb);
2076 goto out;
2077 }
2078
2079 /*
2080 * Write as much data as the socket permits, but no more than a
2081 * a single sndbuf at a time.
2082 */
2083 m->m_len = sbspace(sb);
2084 if (m->m_len > ab->ps.len - job->aio_sent) {
2085 m->m_len = ab->ps.len - job->aio_sent;
2086 moretocome = false;
2087 } else
2088 moretocome = true;
2089 if (m->m_len > toep->params.sndbuf) {
2090 m->m_len = toep->params.sndbuf;
2091 sendmore = true;
2092 } else
2093 sendmore = false;
2094
2095 if (!TAILQ_EMPTY(&toep->aiotx_jobq))
2096 moretocome = true;
2097 SOCKBUF_UNLOCK(sb);
2098 MPASS(m->m_len != 0);
2099
2100 /* Inlined tcp_usr_send(). */
2101
2102 inp = toep->inp;
2103 INP_WLOCK(inp);
2104 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
2105 INP_WUNLOCK(inp);
2106 sbunlock(sb);
2107 error = ECONNRESET;
2108 goto out;
2109 }
2110
2111 refcount_acquire(&ab->refcount);
2112 m_extadd(m, NULL, ab->ps.len, t4_aiotx_mbuf_free, ab,
2113 (void *)(uintptr_t)job->aio_sent, 0, EXT_NET_DRV);
2114 m->m_ext.ext_flags |= EXT_FLAG_AIOTX;
2115 job->aio_sent += m->m_len;
2116
2117 sbappendstream(sb, m, 0);
2118 m = NULL;
2119
2120 if (!(inp->inp_flags & INP_DROPPED)) {
2121 tp = intotcpcb(inp);
2122 if (moretocome)
2123 tp->t_flags |= TF_MORETOCOME;
2124 error = tp->t_fb->tfb_tcp_output(tp);
2125 if (moretocome)
2126 tp->t_flags &= ~TF_MORETOCOME;
2127 }
2128
2129 INP_WUNLOCK(inp);
2130 if (sendmore)
2131 goto sendanother;
2132 sbunlock(sb);
2133
2134 if (error)
2135 goto out;
2136
2137 /*
2138 * If this is a non-blocking socket and the request has not
2139 * been fully completed, requeue it until the socket is ready
2140 * again.
2141 */
2142 if (job->aio_sent < job->uaiocb.aio_nbytes &&
2143 !(so->so_state & SS_NBIO)) {
2144 SOCKBUF_LOCK(sb);
2145 if (!aio_set_cancel_function(job, t4_aiotx_cancel)) {
2146 SOCKBUF_UNLOCK(sb);
2147 error = ECANCELED;
2148 goto out;
2149 }
2150 TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list);
2151 return;
2152 }
2153
2154 /*
2155 * If the request will not be requeued, drop a reference on
2156 * the aiotx buffer. Any mbufs in flight should still
2157 * contain a reference, but this drops the reference that the
2158 * job owns while it is waiting to queue mbufs to the socket.
2159 */
2160 free_aiotx_buffer(ab);
2161
2162 out:
2163 if (error) {
2164 if (ab != NULL) {
2165 job->aio_error = error;
2166 free_aiotx_buffer(ab);
2167 } else {
2168 MPASS(job->aio_sent == 0);
2169 aio_complete(job, -1, error);
2170 }
2171 }
2172 if (m != NULL)
2173 m_free(m);
2174 SOCKBUF_LOCK(sb);
2175 }
2176
2177 static void
t4_aiotx_task(void * context,int pending)2178 t4_aiotx_task(void *context, int pending)
2179 {
2180 struct toepcb *toep = context;
2181 struct socket *so;
2182 struct kaiocb *job;
2183
2184 so = toep->aiotx_so;
2185 CURVNET_SET(toep->vnet);
2186 SOCKBUF_LOCK(&so->so_snd);
2187 while (!TAILQ_EMPTY(&toep->aiotx_jobq) && sowriteable(so)) {
2188 job = TAILQ_FIRST(&toep->aiotx_jobq);
2189 TAILQ_REMOVE(&toep->aiotx_jobq, job, list);
2190 if (!aio_clear_cancel_function(job))
2191 continue;
2192
2193 t4_aiotx_process_job(toep, so, job);
2194 }
2195 toep->aiotx_so = NULL;
2196 SOCKBUF_UNLOCK(&so->so_snd);
2197 CURVNET_RESTORE();
2198
2199 free_toepcb(toep);
2200 SOCK_LOCK(so);
2201 sorele(so);
2202 }
2203
2204 static void
t4_aiotx_queue_toep(struct socket * so,struct toepcb * toep)2205 t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep)
2206 {
2207
2208 SOCKBUF_LOCK_ASSERT(&toep->inp->inp_socket->so_snd);
2209 #ifdef VERBOSE_TRACES
2210 CTR3(KTR_CXGBE, "%s: queueing aiotx task for tid %d, active = %s",
2211 __func__, toep->tid, toep->aiotx_so != NULL ? "true" : "false");
2212 #endif
2213 if (toep->aiotx_so != NULL)
2214 return;
2215 soref(so);
2216 toep->aiotx_so = so;
2217 hold_toepcb(toep);
2218 soaio_enqueue(&toep->aiotx_task);
2219 }
2220
2221 static void
t4_aiotx_cancel(struct kaiocb * job)2222 t4_aiotx_cancel(struct kaiocb *job)
2223 {
2224 struct aiotx_buffer *ab;
2225 struct socket *so;
2226 struct sockbuf *sb;
2227 struct tcpcb *tp;
2228 struct toepcb *toep;
2229
2230 so = job->fd_file->f_data;
2231 tp = so_sototcpcb(so);
2232 toep = tp->t_toe;
2233 MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE);
2234 sb = &so->so_snd;
2235
2236 SOCKBUF_LOCK(sb);
2237 if (!aio_cancel_cleared(job))
2238 TAILQ_REMOVE(&toep->aiotx_jobq, job, list);
2239 SOCKBUF_UNLOCK(sb);
2240
2241 ab = job->backend1;
2242 if (ab != NULL)
2243 free_aiotx_buffer(ab);
2244 else
2245 aio_cancel(job);
2246 }
2247
2248 int
t4_aio_queue_aiotx(struct socket * so,struct kaiocb * job)2249 t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job)
2250 {
2251 struct tcpcb *tp = so_sototcpcb(so);
2252 struct toepcb *toep = tp->t_toe;
2253 struct adapter *sc = td_adapter(toep->td);
2254
2255 /* This only handles writes. */
2256 if (job->uaiocb.aio_lio_opcode != LIO_WRITE)
2257 return (EOPNOTSUPP);
2258
2259 if (!sc->tt.tx_zcopy)
2260 return (EOPNOTSUPP);
2261
2262 if (tls_tx_key(toep))
2263 return (EOPNOTSUPP);
2264
2265 SOCKBUF_LOCK(&so->so_snd);
2266 #ifdef VERBOSE_TRACES
2267 CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid);
2268 #endif
2269 if (!aio_set_cancel_function(job, t4_aiotx_cancel))
2270 panic("new job was cancelled");
2271 TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list);
2272 if (sowriteable(so))
2273 t4_aiotx_queue_toep(so, toep);
2274 SOCKBUF_UNLOCK(&so->so_snd);
2275 return (0);
2276 }
2277
2278 void
aiotx_init_toep(struct toepcb * toep)2279 aiotx_init_toep(struct toepcb *toep)
2280 {
2281
2282 TAILQ_INIT(&toep->aiotx_jobq);
2283 TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep);
2284 }
2285 #endif
2286