1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
5 * All rights reserved.
6 * Written by: Navdeep Parhar <[email protected]>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include "opt_inet.h"
34 #include "opt_inet6.h"
35 #include "opt_kern_tls.h"
36 #include "opt_ratelimit.h"
37
38 #ifdef TCP_OFFLOAD
39 #include <sys/param.h>
40 #include <sys/aio.h>
41 #include <sys/file.h>
42 #include <sys/kernel.h>
43 #include <sys/ktr.h>
44 #include <sys/module.h>
45 #include <sys/proc.h>
46 #include <sys/protosw.h>
47 #include <sys/domain.h>
48 #include <sys/socket.h>
49 #include <sys/socketvar.h>
50 #include <sys/sglist.h>
51 #include <sys/taskqueue.h>
52 #include <netinet/in.h>
53 #include <netinet/in_pcb.h>
54 #include <netinet/ip.h>
55 #include <netinet/ip6.h>
56 #define TCPSTATES
57 #include <netinet/tcp_fsm.h>
58 #include <netinet/tcp_seq.h>
59 #include <netinet/tcp_var.h>
60 #include <netinet/toecore.h>
61
62 #include <security/mac/mac_framework.h>
63
64 #include <vm/vm.h>
65 #include <vm/vm_extern.h>
66 #include <vm/pmap.h>
67 #include <vm/vm_map.h>
68 #include <vm/vm_page.h>
69
70 #include <dev/iscsi/iscsi_proto.h>
71
72 #include "common/common.h"
73 #include "common/t4_msg.h"
74 #include "common/t4_regs.h"
75 #include "common/t4_tcb.h"
76 #include "tom/t4_tom_l2t.h"
77 #include "tom/t4_tom.h"
78
79 static void t4_aiotx_cancel(struct kaiocb *job);
80 static void t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep);
81
82 void
send_flowc_wr(struct toepcb * toep,struct tcpcb * tp)83 send_flowc_wr(struct toepcb *toep, struct tcpcb *tp)
84 {
85 struct wrqe *wr;
86 struct fw_flowc_wr *flowc;
87 unsigned int nparams, flowclen, paramidx;
88 struct vi_info *vi = toep->vi;
89 struct port_info *pi = vi->pi;
90 struct adapter *sc = pi->adapter;
91 unsigned int pfvf = sc->pf << S_FW_VIID_PFN;
92 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
93
94 KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
95 ("%s: flowc for tid %u sent already", __func__, toep->tid));
96
97 if (tp != NULL)
98 nparams = 8;
99 else
100 nparams = 6;
101 if (ulp_mode(toep) == ULP_MODE_TLS)
102 nparams++;
103 if (toep->tls.fcplenmax != 0)
104 nparams++;
105 if (toep->params.tc_idx != -1) {
106 MPASS(toep->params.tc_idx >= 0 &&
107 toep->params.tc_idx < sc->params.nsched_cls);
108 nparams++;
109 }
110
111 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
112
113 wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq);
114 if (wr == NULL) {
115 /* XXX */
116 panic("%s: allocation failure.", __func__);
117 }
118 flowc = wrtod(wr);
119 memset(flowc, 0, wr->wr_len);
120
121 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
122 V_FW_FLOWC_WR_NPARAMS(nparams));
123 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
124 V_FW_WR_FLOWID(toep->tid));
125
126 #define FLOWC_PARAM(__m, __v) \
127 do { \
128 flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \
129 flowc->mnemval[paramidx].val = htobe32(__v); \
130 paramidx++; \
131 } while (0)
132
133 paramidx = 0;
134
135 FLOWC_PARAM(PFNVFN, pfvf);
136 FLOWC_PARAM(CH, pi->tx_chan);
137 FLOWC_PARAM(PORT, pi->tx_chan);
138 FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id);
139 FLOWC_PARAM(SNDBUF, toep->params.sndbuf);
140 if (tp) {
141 FLOWC_PARAM(MSS, toep->params.emss);
142 FLOWC_PARAM(SNDNXT, tp->snd_nxt);
143 FLOWC_PARAM(RCVNXT, tp->rcv_nxt);
144 } else
145 FLOWC_PARAM(MSS, 512);
146 CTR6(KTR_CXGBE,
147 "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x",
148 __func__, toep->tid, toep->params.emss, toep->params.sndbuf,
149 tp ? tp->snd_nxt : 0, tp ? tp->rcv_nxt : 0);
150
151 if (ulp_mode(toep) == ULP_MODE_TLS)
152 FLOWC_PARAM(ULP_MODE, ulp_mode(toep));
153 if (toep->tls.fcplenmax != 0)
154 FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax);
155 if (toep->params.tc_idx != -1)
156 FLOWC_PARAM(SCHEDCLASS, toep->params.tc_idx);
157 #undef FLOWC_PARAM
158
159 KASSERT(paramidx == nparams, ("nparams mismatch"));
160
161 txsd->tx_credits = howmany(flowclen, 16);
162 txsd->plen = 0;
163 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
164 ("%s: not enough credits (%d)", __func__, toep->tx_credits));
165 toep->tx_credits -= txsd->tx_credits;
166 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
167 toep->txsd_pidx = 0;
168 toep->txsd_avail--;
169
170 toep->flags |= TPF_FLOWC_WR_SENT;
171 t4_wrq_tx(sc, wr);
172 }
173
174 #ifdef RATELIMIT
175 /*
176 * Input is Bytes/second (so_max_pacing_rate), chip counts in Kilobits/second.
177 */
178 static int
update_tx_rate_limit(struct adapter * sc,struct toepcb * toep,u_int Bps)179 update_tx_rate_limit(struct adapter *sc, struct toepcb *toep, u_int Bps)
180 {
181 int tc_idx, rc;
182 const u_int kbps = (u_int) (uint64_t)Bps * 8ULL / 1000;
183 const int port_id = toep->vi->pi->port_id;
184
185 CTR3(KTR_CXGBE, "%s: tid %u, rate %uKbps", __func__, toep->tid, kbps);
186
187 if (kbps == 0) {
188 /* unbind */
189 tc_idx = -1;
190 } else {
191 rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx);
192 if (rc != 0)
193 return (rc);
194 MPASS(tc_idx >= 0 && tc_idx < sc->params.nsched_cls);
195 }
196
197 if (toep->params.tc_idx != tc_idx) {
198 struct wrqe *wr;
199 struct fw_flowc_wr *flowc;
200 int nparams = 1, flowclen, flowclen16;
201 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
202
203 flowclen = sizeof(*flowc) + nparams * sizeof(struct
204 fw_flowc_mnemval);
205 flowclen16 = howmany(flowclen, 16);
206 if (toep->tx_credits < flowclen16 || toep->txsd_avail == 0 ||
207 (wr = alloc_wrqe(roundup2(flowclen, 16),
208 &toep->ofld_txq->wrq)) == NULL) {
209 if (tc_idx >= 0)
210 t4_release_cl_rl(sc, port_id, tc_idx);
211 return (ENOMEM);
212 }
213
214 flowc = wrtod(wr);
215 memset(flowc, 0, wr->wr_len);
216
217 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
218 V_FW_FLOWC_WR_NPARAMS(nparams));
219 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(flowclen16) |
220 V_FW_WR_FLOWID(toep->tid));
221
222 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS;
223 if (tc_idx == -1)
224 flowc->mnemval[0].val = htobe32(0xff);
225 else
226 flowc->mnemval[0].val = htobe32(tc_idx);
227
228 txsd->tx_credits = flowclen16;
229 txsd->plen = 0;
230 toep->tx_credits -= txsd->tx_credits;
231 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
232 toep->txsd_pidx = 0;
233 toep->txsd_avail--;
234 t4_wrq_tx(sc, wr);
235 }
236
237 if (toep->params.tc_idx >= 0)
238 t4_release_cl_rl(sc, port_id, toep->params.tc_idx);
239 toep->params.tc_idx = tc_idx;
240
241 return (0);
242 }
243 #endif
244
245 void
send_reset(struct adapter * sc,struct toepcb * toep,uint32_t snd_nxt)246 send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
247 {
248 struct wrqe *wr;
249 struct cpl_abort_req *req;
250 int tid = toep->tid;
251 struct inpcb *inp = toep->inp;
252 struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */
253
254 INP_WLOCK_ASSERT(inp);
255
256 CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s",
257 __func__, toep->tid,
258 inp->inp_flags & INP_DROPPED ? "inp dropped" :
259 tcpstates[tp->t_state],
260 toep->flags, inp->inp_flags,
261 toep->flags & TPF_ABORT_SHUTDOWN ?
262 " (abort already in progress)" : "");
263
264 if (toep->flags & TPF_ABORT_SHUTDOWN)
265 return; /* abort already in progress */
266
267 toep->flags |= TPF_ABORT_SHUTDOWN;
268
269 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
270 ("%s: flowc_wr not sent for tid %d.", __func__, tid));
271
272 wr = alloc_wrqe(sizeof(*req), &toep->ofld_txq->wrq);
273 if (wr == NULL) {
274 /* XXX */
275 panic("%s: allocation failure.", __func__);
276 }
277 req = wrtod(wr);
278
279 INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid);
280 if (inp->inp_flags & INP_DROPPED)
281 req->rsvd0 = htobe32(snd_nxt);
282 else
283 req->rsvd0 = htobe32(tp->snd_nxt);
284 req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT);
285 req->cmd = CPL_ABORT_SEND_RST;
286
287 /*
288 * XXX: What's the correct way to tell that the inp hasn't been detached
289 * from its socket? Should I even be flushing the snd buffer here?
290 */
291 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
292 struct socket *so = inp->inp_socket;
293
294 if (so != NULL) /* because I'm not sure. See comment above */
295 sbflush(&so->so_snd);
296 }
297
298 t4_l2t_send(sc, wr, toep->l2te);
299 }
300
301 /*
302 * Called when a connection is established to translate the TCP options
303 * reported by HW to FreeBSD's native format.
304 */
305 static void
assign_rxopt(struct tcpcb * tp,uint16_t opt)306 assign_rxopt(struct tcpcb *tp, uint16_t opt)
307 {
308 struct toepcb *toep = tp->t_toe;
309 struct inpcb *inp = tp->t_inpcb;
310 struct adapter *sc = td_adapter(toep->td);
311
312 INP_LOCK_ASSERT(inp);
313
314 toep->params.mtu_idx = G_TCPOPT_MSS(opt);
315 tp->t_maxseg = sc->params.mtus[toep->params.mtu_idx];
316 if (inp->inp_inc.inc_flags & INC_ISIPV6)
317 tp->t_maxseg -= sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
318 else
319 tp->t_maxseg -= sizeof(struct ip) + sizeof(struct tcphdr);
320
321 toep->params.emss = tp->t_maxseg;
322 if (G_TCPOPT_TSTAMP(opt)) {
323 toep->params.tstamp = 1;
324 toep->params.emss -= TCPOLEN_TSTAMP_APPA;
325 tp->t_flags |= TF_RCVD_TSTMP; /* timestamps ok */
326 tp->ts_recent = 0; /* hmmm */
327 tp->ts_recent_age = tcp_ts_getticks();
328 } else
329 toep->params.tstamp = 0;
330
331 if (G_TCPOPT_SACK(opt)) {
332 toep->params.sack = 1;
333 tp->t_flags |= TF_SACK_PERMIT; /* should already be set */
334 } else {
335 toep->params.sack = 0;
336 tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */
337 }
338
339 if (G_TCPOPT_WSCALE_OK(opt))
340 tp->t_flags |= TF_RCVD_SCALE;
341
342 /* Doing window scaling? */
343 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
344 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
345 tp->rcv_scale = tp->request_r_scale;
346 tp->snd_scale = G_TCPOPT_SND_WSCALE(opt);
347 } else
348 toep->params.wscale = 0;
349
350 CTR6(KTR_CXGBE,
351 "assign_rxopt: tid %d, mtu_idx %u, emss %u, ts %u, sack %u, wscale %u",
352 toep->tid, toep->params.mtu_idx, toep->params.emss,
353 toep->params.tstamp, toep->params.sack, toep->params.wscale);
354 }
355
356 /*
357 * Completes some final bits of initialization for just established connections
358 * and changes their state to TCPS_ESTABLISHED.
359 *
360 * The ISNs are from the exchange of SYNs.
361 */
362 void
make_established(struct toepcb * toep,uint32_t iss,uint32_t irs,uint16_t opt)363 make_established(struct toepcb *toep, uint32_t iss, uint32_t irs, uint16_t opt)
364 {
365 struct inpcb *inp = toep->inp;
366 struct socket *so = inp->inp_socket;
367 struct tcpcb *tp = intotcpcb(inp);
368 uint16_t tcpopt = be16toh(opt);
369
370 INP_WLOCK_ASSERT(inp);
371 KASSERT(tp->t_state == TCPS_SYN_SENT ||
372 tp->t_state == TCPS_SYN_RECEIVED,
373 ("%s: TCP state %s", __func__, tcpstates[tp->t_state]));
374
375 CTR6(KTR_CXGBE, "%s: tid %d, so %p, inp %p, tp %p, toep %p",
376 __func__, toep->tid, so, inp, tp, toep);
377
378 tcp_state_change(tp, TCPS_ESTABLISHED);
379 tp->t_starttime = ticks;
380 TCPSTAT_INC(tcps_connects);
381
382 tp->irs = irs;
383 tcp_rcvseqinit(tp);
384 tp->rcv_wnd = (u_int)toep->params.opt0_bufsize << 10;
385 tp->rcv_adv += tp->rcv_wnd;
386 tp->last_ack_sent = tp->rcv_nxt;
387
388 tp->iss = iss;
389 tcp_sendseqinit(tp);
390 tp->snd_una = iss + 1;
391 tp->snd_nxt = iss + 1;
392 tp->snd_max = iss + 1;
393
394 assign_rxopt(tp, tcpopt);
395 send_flowc_wr(toep, tp);
396
397 soisconnected(so);
398
399 if (ulp_mode(toep) == ULP_MODE_TLS)
400 tls_establish(toep);
401 }
402
403 int
send_rx_credits(struct adapter * sc,struct toepcb * toep,int credits)404 send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
405 {
406 struct wrqe *wr;
407 struct cpl_rx_data_ack *req;
408 uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
409
410 KASSERT(credits >= 0, ("%s: %d credits", __func__, credits));
411
412 wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
413 if (wr == NULL)
414 return (0);
415 req = wrtod(wr);
416
417 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
418 req->credit_dack = htobe32(dack | V_RX_CREDITS(credits));
419
420 t4_wrq_tx(sc, wr);
421 return (credits);
422 }
423
424 void
send_rx_modulate(struct adapter * sc,struct toepcb * toep)425 send_rx_modulate(struct adapter *sc, struct toepcb *toep)
426 {
427 struct wrqe *wr;
428 struct cpl_rx_data_ack *req;
429
430 wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
431 if (wr == NULL)
432 return;
433 req = wrtod(wr);
434
435 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
436 req->credit_dack = htobe32(F_RX_MODULATE_RX);
437
438 t4_wrq_tx(sc, wr);
439 }
440
441 void
t4_rcvd_locked(struct toedev * tod,struct tcpcb * tp)442 t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp)
443 {
444 struct adapter *sc = tod->tod_softc;
445 struct inpcb *inp = tp->t_inpcb;
446 struct socket *so = inp->inp_socket;
447 struct sockbuf *sb = &so->so_rcv;
448 struct toepcb *toep = tp->t_toe;
449 int rx_credits;
450
451 INP_WLOCK_ASSERT(inp);
452 SOCKBUF_LOCK_ASSERT(sb);
453
454 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
455 if (rx_credits > 0 &&
456 (tp->rcv_wnd <= 32 * 1024 || rx_credits >= 64 * 1024 ||
457 (rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) ||
458 sbused(sb) + tp->rcv_wnd < sb->sb_lowat)) {
459 rx_credits = send_rx_credits(sc, toep, rx_credits);
460 tp->rcv_wnd += rx_credits;
461 tp->rcv_adv += rx_credits;
462 } else if (toep->flags & TPF_FORCE_CREDITS)
463 send_rx_modulate(sc, toep);
464 }
465
466 void
t4_rcvd(struct toedev * tod,struct tcpcb * tp)467 t4_rcvd(struct toedev *tod, struct tcpcb *tp)
468 {
469 struct inpcb *inp = tp->t_inpcb;
470 struct socket *so = inp->inp_socket;
471 struct sockbuf *sb = &so->so_rcv;
472
473 SOCKBUF_LOCK(sb);
474 t4_rcvd_locked(tod, tp);
475 SOCKBUF_UNLOCK(sb);
476 }
477
478 /*
479 * Close a connection by sending a CPL_CLOSE_CON_REQ message.
480 */
481 int
t4_close_conn(struct adapter * sc,struct toepcb * toep)482 t4_close_conn(struct adapter *sc, struct toepcb *toep)
483 {
484 struct wrqe *wr;
485 struct cpl_close_con_req *req;
486 unsigned int tid = toep->tid;
487
488 CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid,
489 toep->flags & TPF_FIN_SENT ? ", IGNORED" : "");
490
491 if (toep->flags & TPF_FIN_SENT)
492 return (0);
493
494 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
495 ("%s: flowc_wr not sent for tid %u.", __func__, tid));
496
497 wr = alloc_wrqe(sizeof(*req), &toep->ofld_txq->wrq);
498 if (wr == NULL) {
499 /* XXX */
500 panic("%s: allocation failure.", __func__);
501 }
502 req = wrtod(wr);
503
504 req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) |
505 V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr)));
506 req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) |
507 V_FW_WR_FLOWID(tid));
508 req->wr.wr_lo = cpu_to_be64(0);
509 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
510 req->rsvd = 0;
511
512 toep->flags |= TPF_FIN_SENT;
513 toep->flags &= ~TPF_SEND_FIN;
514 t4_l2t_send(sc, wr, toep->l2te);
515
516 return (0);
517 }
518
519 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
520 #define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16))
521 #define MIN_ISO_TX_CREDITS (howmany(sizeof(struct cpl_tx_data_iso), 16))
522 #define MIN_TX_CREDITS(iso) \
523 (MIN_OFLD_TX_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0))
524
525 /* Maximum amount of immediate data we could stuff in a WR */
526 static inline int
max_imm_payload(int tx_credits,int iso)527 max_imm_payload(int tx_credits, int iso)
528 {
529 const int iso_cpl_size = iso ? sizeof(struct cpl_tx_data_iso) : 0;
530 const int n = 1; /* Use no more than one desc for imm. data WR */
531
532 KASSERT(tx_credits >= 0 &&
533 tx_credits <= MAX_OFLD_TX_CREDITS,
534 ("%s: %d credits", __func__, tx_credits));
535
536 if (tx_credits < MIN_TX_CREDITS(iso))
537 return (0);
538
539 if (tx_credits >= (n * EQ_ESIZE) / 16)
540 return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr) -
541 iso_cpl_size);
542 else
543 return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr) -
544 iso_cpl_size);
545 }
546
547 /* Maximum number of SGL entries we could stuff in a WR */
548 static inline int
max_dsgl_nsegs(int tx_credits,int iso)549 max_dsgl_nsegs(int tx_credits, int iso)
550 {
551 int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */
552 int sge_pair_credits = tx_credits - MIN_TX_CREDITS(iso);
553
554 KASSERT(tx_credits >= 0 &&
555 tx_credits <= MAX_OFLD_TX_CREDITS,
556 ("%s: %d credits", __func__, tx_credits));
557
558 if (tx_credits < MIN_TX_CREDITS(iso))
559 return (0);
560
561 nseg += 2 * (sge_pair_credits * 16 / 24);
562 if ((sge_pair_credits * 16) % 24 == 16)
563 nseg++;
564
565 return (nseg);
566 }
567
568 static inline void
write_tx_wr(void * dst,struct toepcb * toep,int fw_wr_opcode,unsigned int immdlen,unsigned int plen,uint8_t credits,int shove,int ulp_submode)569 write_tx_wr(void *dst, struct toepcb *toep, int fw_wr_opcode,
570 unsigned int immdlen, unsigned int plen, uint8_t credits, int shove,
571 int ulp_submode)
572 {
573 struct fw_ofld_tx_data_wr *txwr = dst;
574
575 txwr->op_to_immdlen = htobe32(V_WR_OP(fw_wr_opcode) |
576 V_FW_WR_IMMDLEN(immdlen));
577 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
578 V_FW_WR_LEN16(credits));
579 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ulp_mode(toep)) |
580 V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove));
581 txwr->plen = htobe32(plen);
582
583 if (toep->params.tx_align > 0) {
584 if (plen < 2 * toep->params.emss)
585 txwr->lsodisable_to_flags |=
586 htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE);
587 else
588 txwr->lsodisable_to_flags |=
589 htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD |
590 (toep->params.nagle == 0 ? 0 :
591 F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE));
592 }
593 }
594
595 /*
596 * Generate a DSGL from a starting mbuf. The total number of segments and the
597 * maximum segments in any one mbuf are provided.
598 */
599 static void
write_tx_sgl(void * dst,struct mbuf * start,struct mbuf * stop,int nsegs,int n)600 write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n)
601 {
602 struct mbuf *m;
603 struct ulptx_sgl *usgl = dst;
604 int i, j, rc;
605 struct sglist sg;
606 struct sglist_seg segs[n];
607
608 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
609
610 sglist_init(&sg, n, segs);
611 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
612 V_ULPTX_NSGE(nsegs));
613
614 i = -1;
615 for (m = start; m != stop; m = m->m_next) {
616 if (m->m_flags & M_EXTPG)
617 rc = sglist_append_mbuf_epg(&sg, m,
618 mtod(m, vm_offset_t), m->m_len);
619 else
620 rc = sglist_append(&sg, mtod(m, void *), m->m_len);
621 if (__predict_false(rc != 0))
622 panic("%s: sglist_append %d", __func__, rc);
623
624 for (j = 0; j < sg.sg_nseg; i++, j++) {
625 if (i < 0) {
626 usgl->len0 = htobe32(segs[j].ss_len);
627 usgl->addr0 = htobe64(segs[j].ss_paddr);
628 } else {
629 usgl->sge[i / 2].len[i & 1] =
630 htobe32(segs[j].ss_len);
631 usgl->sge[i / 2].addr[i & 1] =
632 htobe64(segs[j].ss_paddr);
633 }
634 #ifdef INVARIANTS
635 nsegs--;
636 #endif
637 }
638 sglist_reset(&sg);
639 }
640 if (i & 1)
641 usgl->sge[i / 2].len[1] = htobe32(0);
642 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p",
643 __func__, nsegs, start, stop));
644 }
645
646 /*
647 * Max number of SGL entries an offload tx work request can have. This is 41
648 * (1 + 40) for a full 512B work request.
649 * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40)
650 */
651 #define OFLD_SGL_LEN (41)
652
653 /*
654 * Send data and/or a FIN to the peer.
655 *
656 * The socket's so_snd buffer consists of a stream of data starting with sb_mb
657 * and linked together with m_next. sb_sndptr, if set, is the last mbuf that
658 * was transmitted.
659 *
660 * drop indicates the number of bytes that should be dropped from the head of
661 * the send buffer. It is an optimization that lets do_fw4_ack avoid creating
662 * contention on the send buffer lock (before this change it used to do
663 * sowwakeup and then t4_push_frames right after that when recovering from tx
664 * stalls). When drop is set this function MUST drop the bytes and wake up any
665 * writers.
666 */
667 void
t4_push_frames(struct adapter * sc,struct toepcb * toep,int drop)668 t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
669 {
670 struct mbuf *sndptr, *m, *sb_sndptr;
671 struct fw_ofld_tx_data_wr *txwr;
672 struct wrqe *wr;
673 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
674 struct inpcb *inp = toep->inp;
675 struct tcpcb *tp = intotcpcb(inp);
676 struct socket *so = inp->inp_socket;
677 struct sockbuf *sb = &so->so_snd;
678 int tx_credits, shove, compl, sowwakeup;
679 struct ofld_tx_sdesc *txsd;
680 bool nomap_mbuf_seen;
681
682 INP_WLOCK_ASSERT(inp);
683 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
684 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
685
686 KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
687 ulp_mode(toep) == ULP_MODE_TCPDDP ||
688 ulp_mode(toep) == ULP_MODE_TLS ||
689 ulp_mode(toep) == ULP_MODE_RDMA,
690 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
691
692 #ifdef VERBOSE_TRACES
693 CTR5(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
694 __func__, toep->tid, toep->flags, tp->t_flags, drop);
695 #endif
696 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
697 return;
698
699 #ifdef RATELIMIT
700 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
701 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
702 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
703 }
704 #endif
705
706 /*
707 * This function doesn't resume by itself. Someone else must clear the
708 * flag and call this function.
709 */
710 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
711 KASSERT(drop == 0,
712 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
713 return;
714 }
715
716 txsd = &toep->txsd[toep->txsd_pidx];
717 do {
718 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
719 max_imm = max_imm_payload(tx_credits, 0);
720 max_nsegs = max_dsgl_nsegs(tx_credits, 0);
721
722 SOCKBUF_LOCK(sb);
723 sowwakeup = drop;
724 if (drop) {
725 sbdrop_locked(sb, drop);
726 drop = 0;
727 }
728 sb_sndptr = sb->sb_sndptr;
729 sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb;
730 plen = 0;
731 nsegs = 0;
732 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
733 nomap_mbuf_seen = false;
734 for (m = sndptr; m != NULL; m = m->m_next) {
735 int n;
736
737 if ((m->m_flags & M_NOTAVAIL) != 0)
738 break;
739 if (m->m_flags & M_EXTPG) {
740 #ifdef KERN_TLS
741 if (m->m_epg_tls != NULL) {
742 toep->flags |= TPF_KTLS;
743 if (plen == 0) {
744 SOCKBUF_UNLOCK(sb);
745 t4_push_ktls(sc, toep, 0);
746 return;
747 }
748 break;
749 }
750 #endif
751 n = sglist_count_mbuf_epg(m,
752 mtod(m, vm_offset_t), m->m_len);
753 } else
754 n = sglist_count(mtod(m, void *), m->m_len);
755
756 nsegs += n;
757 plen += m->m_len;
758
759 /* This mbuf sent us _over_ the nsegs limit, back out */
760 if (plen > max_imm && nsegs > max_nsegs) {
761 nsegs -= n;
762 plen -= m->m_len;
763 if (plen == 0) {
764 /* Too few credits */
765 toep->flags |= TPF_TX_SUSPENDED;
766 if (sowwakeup) {
767 if (!TAILQ_EMPTY(
768 &toep->aiotx_jobq))
769 t4_aiotx_queue_toep(so,
770 toep);
771 sowwakeup_locked(so);
772 } else
773 SOCKBUF_UNLOCK(sb);
774 SOCKBUF_UNLOCK_ASSERT(sb);
775 return;
776 }
777 break;
778 }
779
780 if (m->m_flags & M_EXTPG)
781 nomap_mbuf_seen = true;
782 if (max_nsegs_1mbuf < n)
783 max_nsegs_1mbuf = n;
784 sb_sndptr = m; /* new sb->sb_sndptr if all goes well */
785
786 /* This mbuf put us right at the max_nsegs limit */
787 if (plen > max_imm && nsegs == max_nsegs) {
788 m = m->m_next;
789 break;
790 }
791 }
792
793 if (sbused(sb) > sb->sb_hiwat * 5 / 8 &&
794 toep->plen_nocompl + plen >= sb->sb_hiwat / 4)
795 compl = 1;
796 else
797 compl = 0;
798
799 if (sb->sb_flags & SB_AUTOSIZE &&
800 V_tcp_do_autosndbuf &&
801 sb->sb_hiwat < V_tcp_autosndbuf_max &&
802 sbused(sb) >= sb->sb_hiwat * 7 / 8) {
803 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
804 V_tcp_autosndbuf_max);
805
806 if (!sbreserve_locked(sb, newsize, so, NULL))
807 sb->sb_flags &= ~SB_AUTOSIZE;
808 else
809 sowwakeup = 1; /* room available */
810 }
811 if (sowwakeup) {
812 if (!TAILQ_EMPTY(&toep->aiotx_jobq))
813 t4_aiotx_queue_toep(so, toep);
814 sowwakeup_locked(so);
815 } else
816 SOCKBUF_UNLOCK(sb);
817 SOCKBUF_UNLOCK_ASSERT(sb);
818
819 /* nothing to send */
820 if (plen == 0) {
821 KASSERT(m == NULL || (m->m_flags & M_NOTAVAIL) != 0,
822 ("%s: nothing to send, but m != NULL is ready",
823 __func__));
824 break;
825 }
826
827 if (__predict_false(toep->flags & TPF_FIN_SENT))
828 panic("%s: excess tx.", __func__);
829
830 shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
831 if (plen <= max_imm && !nomap_mbuf_seen) {
832
833 /* Immediate data tx */
834
835 wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
836 &toep->ofld_txq->wrq);
837 if (wr == NULL) {
838 /* XXX: how will we recover from this? */
839 toep->flags |= TPF_TX_SUSPENDED;
840 return;
841 }
842 txwr = wrtod(wr);
843 credits = howmany(wr->wr_len, 16);
844 write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, plen, plen,
845 credits, shove, 0);
846 m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
847 nsegs = 0;
848 } else {
849 int wr_len;
850
851 /* DSGL tx */
852
853 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
854 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
855 wr = alloc_wrqe(roundup2(wr_len, 16),
856 &toep->ofld_txq->wrq);
857 if (wr == NULL) {
858 /* XXX: how will we recover from this? */
859 toep->flags |= TPF_TX_SUSPENDED;
860 return;
861 }
862 txwr = wrtod(wr);
863 credits = howmany(wr_len, 16);
864 write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, 0, plen,
865 credits, shove, 0);
866 write_tx_sgl(txwr + 1, sndptr, m, nsegs,
867 max_nsegs_1mbuf);
868 if (wr_len & 0xf) {
869 uint64_t *pad = (uint64_t *)
870 ((uintptr_t)txwr + wr_len);
871 *pad = 0;
872 }
873 }
874
875 KASSERT(toep->tx_credits >= credits,
876 ("%s: not enough credits", __func__));
877
878 toep->tx_credits -= credits;
879 toep->tx_nocompl += credits;
880 toep->plen_nocompl += plen;
881 if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
882 toep->tx_nocompl >= toep->tx_total / 4)
883 compl = 1;
884
885 if (compl || ulp_mode(toep) == ULP_MODE_RDMA) {
886 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
887 toep->tx_nocompl = 0;
888 toep->plen_nocompl = 0;
889 }
890
891 tp->snd_nxt += plen;
892 tp->snd_max += plen;
893
894 SOCKBUF_LOCK(sb);
895 KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__));
896 sb->sb_sndptr = sb_sndptr;
897 SOCKBUF_UNLOCK(sb);
898
899 toep->flags |= TPF_TX_DATA_SENT;
900 if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
901 toep->flags |= TPF_TX_SUSPENDED;
902
903 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
904 txsd->plen = plen;
905 txsd->tx_credits = credits;
906 txsd++;
907 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
908 toep->txsd_pidx = 0;
909 txsd = &toep->txsd[0];
910 }
911 toep->txsd_avail--;
912
913 t4_l2t_send(sc, wr, toep->l2te);
914 } while (m != NULL && (m->m_flags & M_NOTAVAIL) == 0);
915
916 /* Send a FIN if requested, but only if there's no more data to send */
917 if (m == NULL && toep->flags & TPF_SEND_FIN)
918 t4_close_conn(sc, toep);
919 }
920
921 static inline void
rqdrop_locked(struct mbufq * q,int plen)922 rqdrop_locked(struct mbufq *q, int plen)
923 {
924 struct mbuf *m;
925
926 while (plen > 0) {
927 m = mbufq_dequeue(q);
928
929 /* Too many credits. */
930 MPASS(m != NULL);
931 M_ASSERTPKTHDR(m);
932
933 /* Partial credits. */
934 MPASS(plen >= m->m_pkthdr.len);
935
936 plen -= m->m_pkthdr.len;
937 m_freem(m);
938 }
939 }
940
941 /*
942 * Not a bit in the TCB, but is a bit in the ulp_submode field of the
943 * CPL_TX_DATA flags field in FW_ISCSI_TX_DATA_WR.
944 */
945 #define ULP_ISO G_TX_ULP_SUBMODE(F_FW_ISCSI_TX_DATA_WR_ULPSUBMODE_ISO)
946
947 static void
write_tx_data_iso(void * dst,u_int ulp_submode,uint8_t flags,uint16_t mss,int len,int npdu)948 write_tx_data_iso(void *dst, u_int ulp_submode, uint8_t flags, uint16_t mss,
949 int len, int npdu)
950 {
951 struct cpl_tx_data_iso *cpl;
952 unsigned int burst_size;
953 unsigned int last;
954
955 /*
956 * The firmware will set the 'F' bit on the last PDU when
957 * either condition is true:
958 *
959 * - this large PDU is marked as the "last" slice
960 *
961 * - the amount of data payload bytes equals the burst_size
962 *
963 * The strategy used here is to always set the burst_size
964 * artificially high (len includes the size of the template
965 * BHS) and only set the "last" flag if the original PDU had
966 * 'F' set.
967 */
968 burst_size = len;
969 last = !!(flags & CXGBE_ISO_F);
970
971 cpl = (struct cpl_tx_data_iso *)dst;
972 cpl->op_to_scsi = htonl(V_CPL_TX_DATA_ISO_OP(CPL_TX_DATA_ISO) |
973 V_CPL_TX_DATA_ISO_FIRST(1) | V_CPL_TX_DATA_ISO_LAST(last) |
974 V_CPL_TX_DATA_ISO_CPLHDRLEN(0) |
975 V_CPL_TX_DATA_ISO_HDRCRC(!!(ulp_submode & ULP_CRC_HEADER)) |
976 V_CPL_TX_DATA_ISO_PLDCRC(!!(ulp_submode & ULP_CRC_DATA)) |
977 V_CPL_TX_DATA_ISO_IMMEDIATE(0) |
978 V_CPL_TX_DATA_ISO_SCSI(CXGBE_ISO_TYPE(flags)));
979
980 cpl->ahs_len = 0;
981 cpl->mpdu = htons(DIV_ROUND_UP(mss, 4));
982 cpl->burst_size = htonl(DIV_ROUND_UP(burst_size, 4));
983 cpl->len = htonl(len);
984 cpl->reserved2_seglen_offset = htonl(0);
985 cpl->datasn_offset = htonl(0);
986 cpl->buffer_offset = htonl(0);
987 cpl->reserved3 = 0;
988 }
989
990 static struct wrqe *
write_iscsi_mbuf_wr(struct toepcb * toep,struct mbuf * sndptr)991 write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
992 {
993 struct mbuf *m;
994 struct fw_ofld_tx_data_wr *txwr;
995 struct cpl_tx_data_iso *cpl_iso;
996 void *p;
997 struct wrqe *wr;
998 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
999 u_int adjusted_plen, imm_data, ulp_submode;
1000 struct inpcb *inp = toep->inp;
1001 struct tcpcb *tp = intotcpcb(inp);
1002 int tx_credits, shove, npdu, wr_len;
1003 uint16_t iso_mss;
1004 static const u_int ulp_extra_len[] = {0, 4, 4, 8};
1005 bool iso;
1006
1007 M_ASSERTPKTHDR(sndptr);
1008
1009 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
1010 if (mbuf_raw_wr(sndptr)) {
1011 plen = sndptr->m_pkthdr.len;
1012 KASSERT(plen <= SGE_MAX_WR_LEN,
1013 ("raw WR len %u is greater than max WR len", plen));
1014 if (plen > tx_credits * 16)
1015 return (NULL);
1016
1017 wr = alloc_wrqe(roundup2(plen, 16), &toep->ofld_txq->wrq);
1018 if (__predict_false(wr == NULL))
1019 return (NULL);
1020
1021 m_copydata(sndptr, 0, plen, wrtod(wr));
1022 return (wr);
1023 }
1024
1025 iso = mbuf_iscsi_iso(sndptr);
1026 max_imm = max_imm_payload(tx_credits, iso);
1027 max_nsegs = max_dsgl_nsegs(tx_credits, iso);
1028 iso_mss = mbuf_iscsi_iso_mss(sndptr);
1029
1030 plen = 0;
1031 nsegs = 0;
1032 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
1033 for (m = sndptr; m != NULL; m = m->m_next) {
1034 int n = sglist_count(mtod(m, void *), m->m_len);
1035
1036 nsegs += n;
1037 plen += m->m_len;
1038
1039 /*
1040 * This mbuf would send us _over_ the nsegs limit.
1041 * Suspend tx because the PDU can't be sent out.
1042 */
1043 if (plen > max_imm && nsegs > max_nsegs)
1044 return (NULL);
1045
1046 if (max_nsegs_1mbuf < n)
1047 max_nsegs_1mbuf = n;
1048 }
1049
1050 if (__predict_false(toep->flags & TPF_FIN_SENT))
1051 panic("%s: excess tx.", __func__);
1052
1053 /*
1054 * We have a PDU to send. All of it goes out in one WR so 'm'
1055 * is NULL. A PDU's length is always a multiple of 4.
1056 */
1057 MPASS(m == NULL);
1058 MPASS((plen & 3) == 0);
1059 MPASS(sndptr->m_pkthdr.len == plen);
1060
1061 shove = !(tp->t_flags & TF_MORETOCOME);
1062
1063 /*
1064 * plen doesn't include header and data digests, which are
1065 * generated and inserted in the right places by the TOE, but
1066 * they do occupy TCP sequence space and need to be accounted
1067 * for.
1068 */
1069 ulp_submode = mbuf_ulp_submode(sndptr);
1070 MPASS(ulp_submode < nitems(ulp_extra_len));
1071 npdu = iso ? howmany(plen - ISCSI_BHS_SIZE, iso_mss) : 1;
1072 adjusted_plen = plen + ulp_extra_len[ulp_submode] * npdu;
1073 if (iso)
1074 adjusted_plen += ISCSI_BHS_SIZE * (npdu - 1);
1075 wr_len = sizeof(*txwr);
1076 if (iso)
1077 wr_len += sizeof(struct cpl_tx_data_iso);
1078 if (plen <= max_imm) {
1079 /* Immediate data tx */
1080 imm_data = plen;
1081 wr_len += plen;
1082 nsegs = 0;
1083 } else {
1084 /* DSGL tx */
1085 imm_data = 0;
1086 wr_len += sizeof(struct ulptx_sgl) +
1087 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
1088 }
1089
1090 wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq);
1091 if (wr == NULL) {
1092 /* XXX: how will we recover from this? */
1093 return (NULL);
1094 }
1095 txwr = wrtod(wr);
1096 credits = howmany(wr->wr_len, 16);
1097
1098 if (iso) {
1099 write_tx_wr(txwr, toep, FW_ISCSI_TX_DATA_WR,
1100 imm_data + sizeof(struct cpl_tx_data_iso),
1101 adjusted_plen, credits, shove, ulp_submode | ULP_ISO);
1102 cpl_iso = (struct cpl_tx_data_iso *)(txwr + 1);
1103 MPASS(plen == sndptr->m_pkthdr.len);
1104 write_tx_data_iso(cpl_iso, ulp_submode,
1105 mbuf_iscsi_iso_flags(sndptr), iso_mss, plen, npdu);
1106 p = cpl_iso + 1;
1107 } else {
1108 write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, imm_data,
1109 adjusted_plen, credits, shove, ulp_submode);
1110 p = txwr + 1;
1111 }
1112
1113 if (imm_data != 0) {
1114 m_copydata(sndptr, 0, plen, p);
1115 } else {
1116 write_tx_sgl(p, sndptr, m, nsegs, max_nsegs_1mbuf);
1117 if (wr_len & 0xf) {
1118 uint64_t *pad = (uint64_t *)((uintptr_t)txwr + wr_len);
1119 *pad = 0;
1120 }
1121 }
1122
1123 KASSERT(toep->tx_credits >= credits,
1124 ("%s: not enough credits: credits %u "
1125 "toep->tx_credits %u tx_credits %u nsegs %u "
1126 "max_nsegs %u iso %d", __func__, credits,
1127 toep->tx_credits, tx_credits, nsegs, max_nsegs, iso));
1128
1129 tp->snd_nxt += adjusted_plen;
1130 tp->snd_max += adjusted_plen;
1131
1132 counter_u64_add(toep->ofld_txq->tx_iscsi_pdus, npdu);
1133 counter_u64_add(toep->ofld_txq->tx_iscsi_octets, plen);
1134 if (iso)
1135 counter_u64_add(toep->ofld_txq->tx_iscsi_iso_wrs, 1);
1136
1137 return (wr);
1138 }
1139
1140 void
t4_push_pdus(struct adapter * sc,struct toepcb * toep,int drop)1141 t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
1142 {
1143 struct mbuf *sndptr, *m;
1144 struct fw_wr_hdr *wrhdr;
1145 struct wrqe *wr;
1146 u_int plen, credits;
1147 struct inpcb *inp = toep->inp;
1148 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
1149 struct mbufq *pduq = &toep->ulp_pduq;
1150
1151 INP_WLOCK_ASSERT(inp);
1152 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
1153 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
1154 KASSERT(ulp_mode(toep) == ULP_MODE_ISCSI,
1155 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
1156
1157 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
1158 return;
1159
1160 /*
1161 * This function doesn't resume by itself. Someone else must clear the
1162 * flag and call this function.
1163 */
1164 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
1165 KASSERT(drop == 0,
1166 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
1167 return;
1168 }
1169
1170 if (drop) {
1171 struct socket *so = inp->inp_socket;
1172 struct sockbuf *sb = &so->so_snd;
1173 int sbu;
1174
1175 /*
1176 * An unlocked read is ok here as the data should only
1177 * transition from a non-zero value to either another
1178 * non-zero value or zero. Once it is zero it should
1179 * stay zero.
1180 */
1181 if (__predict_false(sbused(sb)) > 0) {
1182 SOCKBUF_LOCK(sb);
1183 sbu = sbused(sb);
1184 if (sbu > 0) {
1185 /*
1186 * The data transmitted before the
1187 * tid's ULP mode changed to ISCSI is
1188 * still in so_snd. Incoming credits
1189 * should account for so_snd first.
1190 */
1191 sbdrop_locked(sb, min(sbu, drop));
1192 drop -= min(sbu, drop);
1193 }
1194 sowwakeup_locked(so); /* unlocks so_snd */
1195 }
1196 rqdrop_locked(&toep->ulp_pdu_reclaimq, drop);
1197 }
1198
1199 while ((sndptr = mbufq_first(pduq)) != NULL) {
1200 wr = write_iscsi_mbuf_wr(toep, sndptr);
1201 if (wr == NULL) {
1202 toep->flags |= TPF_TX_SUSPENDED;
1203 return;
1204 }
1205
1206 plen = sndptr->m_pkthdr.len;
1207 credits = howmany(wr->wr_len, 16);
1208 KASSERT(toep->tx_credits >= credits,
1209 ("%s: not enough credits", __func__));
1210
1211 m = mbufq_dequeue(pduq);
1212 MPASS(m == sndptr);
1213 mbufq_enqueue(&toep->ulp_pdu_reclaimq, m);
1214
1215 toep->tx_credits -= credits;
1216 toep->tx_nocompl += credits;
1217 toep->plen_nocompl += plen;
1218
1219 /*
1220 * Ensure there are enough credits for a full-sized WR
1221 * as page pod WRs can be full-sized.
1222 */
1223 if (toep->tx_credits <= SGE_MAX_WR_LEN * 5 / 4 &&
1224 toep->tx_nocompl >= toep->tx_total / 4) {
1225 wrhdr = wrtod(wr);
1226 wrhdr->hi |= htobe32(F_FW_WR_COMPL);
1227 toep->tx_nocompl = 0;
1228 toep->plen_nocompl = 0;
1229 }
1230
1231 toep->flags |= TPF_TX_DATA_SENT;
1232 if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
1233 toep->flags |= TPF_TX_SUSPENDED;
1234
1235 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
1236 txsd->plen = plen;
1237 txsd->tx_credits = credits;
1238 txsd++;
1239 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
1240 toep->txsd_pidx = 0;
1241 txsd = &toep->txsd[0];
1242 }
1243 toep->txsd_avail--;
1244
1245 t4_l2t_send(sc, wr, toep->l2te);
1246 }
1247
1248 /* Send a FIN if requested, but only if there are no more PDUs to send */
1249 if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
1250 t4_close_conn(sc, toep);
1251 }
1252
1253 static inline void
t4_push_data(struct adapter * sc,struct toepcb * toep,int drop)1254 t4_push_data(struct adapter *sc, struct toepcb *toep, int drop)
1255 {
1256
1257 if (ulp_mode(toep) == ULP_MODE_ISCSI)
1258 t4_push_pdus(sc, toep, drop);
1259 else if (tls_tx_key(toep) && toep->tls.mode == TLS_MODE_TLSOM)
1260 t4_push_tls_records(sc, toep, drop);
1261 #ifdef KERN_TLS
1262 else if (toep->flags & TPF_KTLS)
1263 t4_push_ktls(sc, toep, drop);
1264 #endif
1265 else
1266 t4_push_frames(sc, toep, drop);
1267 }
1268
1269 int
t4_tod_output(struct toedev * tod,struct tcpcb * tp)1270 t4_tod_output(struct toedev *tod, struct tcpcb *tp)
1271 {
1272 struct adapter *sc = tod->tod_softc;
1273 #ifdef INVARIANTS
1274 struct inpcb *inp = tp->t_inpcb;
1275 #endif
1276 struct toepcb *toep = tp->t_toe;
1277
1278 INP_WLOCK_ASSERT(inp);
1279 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1280 ("%s: inp %p dropped.", __func__, inp));
1281 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1282
1283 t4_push_data(sc, toep, 0);
1284
1285 return (0);
1286 }
1287
1288 int
t4_send_fin(struct toedev * tod,struct tcpcb * tp)1289 t4_send_fin(struct toedev *tod, struct tcpcb *tp)
1290 {
1291 struct adapter *sc = tod->tod_softc;
1292 #ifdef INVARIANTS
1293 struct inpcb *inp = tp->t_inpcb;
1294 #endif
1295 struct toepcb *toep = tp->t_toe;
1296
1297 INP_WLOCK_ASSERT(inp);
1298 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1299 ("%s: inp %p dropped.", __func__, inp));
1300 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1301
1302 toep->flags |= TPF_SEND_FIN;
1303 if (tp->t_state >= TCPS_ESTABLISHED)
1304 t4_push_data(sc, toep, 0);
1305
1306 return (0);
1307 }
1308
1309 int
t4_send_rst(struct toedev * tod,struct tcpcb * tp)1310 t4_send_rst(struct toedev *tod, struct tcpcb *tp)
1311 {
1312 struct adapter *sc = tod->tod_softc;
1313 #if defined(INVARIANTS)
1314 struct inpcb *inp = tp->t_inpcb;
1315 #endif
1316 struct toepcb *toep = tp->t_toe;
1317
1318 INP_WLOCK_ASSERT(inp);
1319 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1320 ("%s: inp %p dropped.", __func__, inp));
1321 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1322
1323 /* hmmmm */
1324 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
1325 ("%s: flowc for tid %u [%s] not sent already",
1326 __func__, toep->tid, tcpstates[tp->t_state]));
1327
1328 send_reset(sc, toep, 0);
1329 return (0);
1330 }
1331
1332 /*
1333 * Peer has sent us a FIN.
1334 */
1335 static int
do_peer_close(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1336 do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1337 {
1338 struct adapter *sc = iq->adapter;
1339 const struct cpl_peer_close *cpl = (const void *)(rss + 1);
1340 unsigned int tid = GET_TID(cpl);
1341 struct toepcb *toep = lookup_tid(sc, tid);
1342 struct inpcb *inp = toep->inp;
1343 struct tcpcb *tp = NULL;
1344 struct socket *so;
1345 struct epoch_tracker et;
1346 #ifdef INVARIANTS
1347 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1348 #endif
1349
1350 KASSERT(opcode == CPL_PEER_CLOSE,
1351 ("%s: unexpected opcode 0x%x", __func__, opcode));
1352 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1353
1354 if (__predict_false(toep->flags & TPF_SYNQE)) {
1355 /*
1356 * do_pass_establish must have run before do_peer_close and if
1357 * this is still a synqe instead of a toepcb then the connection
1358 * must be getting aborted.
1359 */
1360 MPASS(toep->flags & TPF_ABORT_SHUTDOWN);
1361 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
1362 toep, toep->flags);
1363 return (0);
1364 }
1365
1366 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1367
1368 CURVNET_SET(toep->vnet);
1369 NET_EPOCH_ENTER(et);
1370 INP_WLOCK(inp);
1371 tp = intotcpcb(inp);
1372
1373 CTR6(KTR_CXGBE,
1374 "%s: tid %u (%s), toep_flags 0x%x, ddp_flags 0x%x, inp %p",
1375 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
1376 toep->ddp.flags, inp);
1377
1378 if (toep->flags & TPF_ABORT_SHUTDOWN)
1379 goto done;
1380
1381 so = inp->inp_socket;
1382 socantrcvmore(so);
1383 if (ulp_mode(toep) == ULP_MODE_TCPDDP) {
1384 DDP_LOCK(toep);
1385 if (__predict_false(toep->ddp.flags &
1386 (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)))
1387 handle_ddp_close(toep, tp, cpl->rcv_nxt);
1388 DDP_UNLOCK(toep);
1389 }
1390
1391 if (ulp_mode(toep) == ULP_MODE_RDMA ||
1392 (ulp_mode(toep) == ULP_MODE_ISCSI && chip_id(sc) >= CHELSIO_T6)) {
1393 /*
1394 * There might be data received via DDP before the FIN
1395 * not reported to the driver. Just assume the
1396 * sequence number in the CPL is correct as the
1397 * sequence number of the FIN.
1398 */
1399 } else {
1400 KASSERT(tp->rcv_nxt + 1 == be32toh(cpl->rcv_nxt),
1401 ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
1402 be32toh(cpl->rcv_nxt)));
1403 }
1404
1405 tp->rcv_nxt = be32toh(cpl->rcv_nxt);
1406
1407 switch (tp->t_state) {
1408 case TCPS_SYN_RECEIVED:
1409 tp->t_starttime = ticks;
1410 /* FALLTHROUGH */
1411
1412 case TCPS_ESTABLISHED:
1413 tcp_state_change(tp, TCPS_CLOSE_WAIT);
1414 break;
1415
1416 case TCPS_FIN_WAIT_1:
1417 tcp_state_change(tp, TCPS_CLOSING);
1418 break;
1419
1420 case TCPS_FIN_WAIT_2:
1421 restore_so_proto(so, inp->inp_vflag & INP_IPV6);
1422 tcp_twstart(tp);
1423 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */
1424 NET_EPOCH_EXIT(et);
1425 CURVNET_RESTORE();
1426
1427 INP_WLOCK(inp);
1428 final_cpl_received(toep);
1429 return (0);
1430
1431 default:
1432 log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n",
1433 __func__, tid, tp->t_state);
1434 }
1435 done:
1436 INP_WUNLOCK(inp);
1437 NET_EPOCH_EXIT(et);
1438 CURVNET_RESTORE();
1439 return (0);
1440 }
1441
1442 /*
1443 * Peer has ACK'd our FIN.
1444 */
1445 static int
do_close_con_rpl(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1446 do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss,
1447 struct mbuf *m)
1448 {
1449 struct adapter *sc = iq->adapter;
1450 const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1);
1451 unsigned int tid = GET_TID(cpl);
1452 struct toepcb *toep = lookup_tid(sc, tid);
1453 struct inpcb *inp = toep->inp;
1454 struct tcpcb *tp = NULL;
1455 struct socket *so = NULL;
1456 struct epoch_tracker et;
1457 #ifdef INVARIANTS
1458 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1459 #endif
1460
1461 KASSERT(opcode == CPL_CLOSE_CON_RPL,
1462 ("%s: unexpected opcode 0x%x", __func__, opcode));
1463 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1464 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1465
1466 CURVNET_SET(toep->vnet);
1467 NET_EPOCH_ENTER(et);
1468 INP_WLOCK(inp);
1469 tp = intotcpcb(inp);
1470
1471 CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x",
1472 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags);
1473
1474 if (toep->flags & TPF_ABORT_SHUTDOWN)
1475 goto done;
1476
1477 so = inp->inp_socket;
1478 tp->snd_una = be32toh(cpl->snd_nxt) - 1; /* exclude FIN */
1479
1480 switch (tp->t_state) {
1481 case TCPS_CLOSING: /* see TCPS_FIN_WAIT_2 in do_peer_close too */
1482 restore_so_proto(so, inp->inp_vflag & INP_IPV6);
1483 tcp_twstart(tp);
1484 release:
1485 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */
1486 NET_EPOCH_EXIT(et);
1487 CURVNET_RESTORE();
1488
1489 INP_WLOCK(inp);
1490 final_cpl_received(toep); /* no more CPLs expected */
1491
1492 return (0);
1493 case TCPS_LAST_ACK:
1494 if (tcp_close(tp))
1495 INP_WUNLOCK(inp);
1496 goto release;
1497
1498 case TCPS_FIN_WAIT_1:
1499 if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
1500 soisdisconnected(so);
1501 tcp_state_change(tp, TCPS_FIN_WAIT_2);
1502 break;
1503
1504 default:
1505 log(LOG_ERR,
1506 "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n",
1507 __func__, tid, tcpstates[tp->t_state]);
1508 }
1509 done:
1510 INP_WUNLOCK(inp);
1511 NET_EPOCH_EXIT(et);
1512 CURVNET_RESTORE();
1513 return (0);
1514 }
1515
1516 void
send_abort_rpl(struct adapter * sc,struct sge_ofld_txq * ofld_txq,int tid,int rst_status)1517 send_abort_rpl(struct adapter *sc, struct sge_ofld_txq *ofld_txq, int tid,
1518 int rst_status)
1519 {
1520 struct wrqe *wr;
1521 struct cpl_abort_rpl *cpl;
1522
1523 wr = alloc_wrqe(sizeof(*cpl), &ofld_txq->wrq);
1524 if (wr == NULL) {
1525 /* XXX */
1526 panic("%s: allocation failure.", __func__);
1527 }
1528 cpl = wrtod(wr);
1529
1530 INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid);
1531 cpl->cmd = rst_status;
1532
1533 t4_wrq_tx(sc, wr);
1534 }
1535
1536 static int
abort_status_to_errno(struct tcpcb * tp,unsigned int abort_reason)1537 abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason)
1538 {
1539 switch (abort_reason) {
1540 case CPL_ERR_BAD_SYN:
1541 case CPL_ERR_CONN_RESET:
1542 return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET);
1543 case CPL_ERR_XMIT_TIMEDOUT:
1544 case CPL_ERR_PERSIST_TIMEDOUT:
1545 case CPL_ERR_FINWAIT2_TIMEDOUT:
1546 case CPL_ERR_KEEPALIVE_TIMEDOUT:
1547 return (ETIMEDOUT);
1548 default:
1549 return (EIO);
1550 }
1551 }
1552
1553 /*
1554 * TCP RST from the peer, timeout, or some other such critical error.
1555 */
1556 static int
do_abort_req(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1557 do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1558 {
1559 struct adapter *sc = iq->adapter;
1560 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
1561 unsigned int tid = GET_TID(cpl);
1562 struct toepcb *toep = lookup_tid(sc, tid);
1563 struct sge_ofld_txq *ofld_txq = toep->ofld_txq;
1564 struct inpcb *inp;
1565 struct tcpcb *tp;
1566 struct epoch_tracker et;
1567 #ifdef INVARIANTS
1568 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1569 #endif
1570
1571 KASSERT(opcode == CPL_ABORT_REQ_RSS,
1572 ("%s: unexpected opcode 0x%x", __func__, opcode));
1573 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1574
1575 if (toep->flags & TPF_SYNQE)
1576 return (do_abort_req_synqe(iq, rss, m));
1577
1578 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1579
1580 if (negative_advice(cpl->status)) {
1581 CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)",
1582 __func__, cpl->status, tid, toep->flags);
1583 return (0); /* Ignore negative advice */
1584 }
1585
1586 inp = toep->inp;
1587 CURVNET_SET(toep->vnet);
1588 NET_EPOCH_ENTER(et); /* for tcp_close */
1589 INP_WLOCK(inp);
1590
1591 tp = intotcpcb(inp);
1592
1593 CTR6(KTR_CXGBE,
1594 "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d",
1595 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
1596 inp->inp_flags, cpl->status);
1597
1598 /*
1599 * If we'd initiated an abort earlier the reply to it is responsible for
1600 * cleaning up resources. Otherwise we tear everything down right here
1601 * right now. We owe the T4 a CPL_ABORT_RPL no matter what.
1602 */
1603 if (toep->flags & TPF_ABORT_SHUTDOWN) {
1604 INP_WUNLOCK(inp);
1605 goto done;
1606 }
1607 toep->flags |= TPF_ABORT_SHUTDOWN;
1608
1609 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
1610 struct socket *so = inp->inp_socket;
1611
1612 if (so != NULL)
1613 so_error_set(so, abort_status_to_errno(tp,
1614 cpl->status));
1615 tp = tcp_close(tp);
1616 if (tp == NULL)
1617 INP_WLOCK(inp); /* re-acquire */
1618 }
1619
1620 final_cpl_received(toep);
1621 done:
1622 NET_EPOCH_EXIT(et);
1623 CURVNET_RESTORE();
1624 send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST);
1625 return (0);
1626 }
1627
1628 /*
1629 * Reply to the CPL_ABORT_REQ (send_reset)
1630 */
1631 static int
do_abort_rpl(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1632 do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1633 {
1634 struct adapter *sc = iq->adapter;
1635 const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
1636 unsigned int tid = GET_TID(cpl);
1637 struct toepcb *toep = lookup_tid(sc, tid);
1638 struct inpcb *inp = toep->inp;
1639 #ifdef INVARIANTS
1640 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1641 #endif
1642
1643 KASSERT(opcode == CPL_ABORT_RPL_RSS,
1644 ("%s: unexpected opcode 0x%x", __func__, opcode));
1645 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1646
1647 if (toep->flags & TPF_SYNQE)
1648 return (do_abort_rpl_synqe(iq, rss, m));
1649
1650 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1651
1652 CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d",
1653 __func__, tid, toep, inp, cpl->status);
1654
1655 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1656 ("%s: wasn't expecting abort reply", __func__));
1657
1658 INP_WLOCK(inp);
1659 final_cpl_received(toep);
1660
1661 return (0);
1662 }
1663
1664 static int
do_rx_data(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1665 do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1666 {
1667 struct adapter *sc = iq->adapter;
1668 const struct cpl_rx_data *cpl = mtod(m, const void *);
1669 unsigned int tid = GET_TID(cpl);
1670 struct toepcb *toep = lookup_tid(sc, tid);
1671 struct inpcb *inp = toep->inp;
1672 struct tcpcb *tp;
1673 struct socket *so;
1674 struct sockbuf *sb;
1675 struct epoch_tracker et;
1676 int len, rx_credits;
1677 uint32_t ddp_placed = 0;
1678
1679 if (__predict_false(toep->flags & TPF_SYNQE)) {
1680 /*
1681 * do_pass_establish must have run before do_rx_data and if this
1682 * is still a synqe instead of a toepcb then the connection must
1683 * be getting aborted.
1684 */
1685 MPASS(toep->flags & TPF_ABORT_SHUTDOWN);
1686 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
1687 toep, toep->flags);
1688 m_freem(m);
1689 return (0);
1690 }
1691
1692 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1693
1694 /* strip off CPL header */
1695 m_adj(m, sizeof(*cpl));
1696 len = m->m_pkthdr.len;
1697
1698 INP_WLOCK(inp);
1699 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
1700 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
1701 __func__, tid, len, inp->inp_flags);
1702 INP_WUNLOCK(inp);
1703 m_freem(m);
1704 return (0);
1705 }
1706
1707 tp = intotcpcb(inp);
1708
1709 if (__predict_false(ulp_mode(toep) == ULP_MODE_TLS &&
1710 toep->flags & TPF_TLS_RECEIVE)) {
1711 /* Received "raw" data on a TLS socket. */
1712 CTR3(KTR_CXGBE, "%s: tid %u, raw TLS data (%d bytes)",
1713 __func__, tid, len);
1714 do_rx_data_tls(cpl, toep, m);
1715 return (0);
1716 }
1717
1718 if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq)))
1719 ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt;
1720
1721 tp->rcv_nxt += len;
1722 if (tp->rcv_wnd < len) {
1723 KASSERT(ulp_mode(toep) == ULP_MODE_RDMA,
1724 ("%s: negative window size", __func__));
1725 }
1726
1727 tp->rcv_wnd -= len;
1728 tp->t_rcvtime = ticks;
1729
1730 if (ulp_mode(toep) == ULP_MODE_TCPDDP)
1731 DDP_LOCK(toep);
1732 so = inp_inpcbtosocket(inp);
1733 sb = &so->so_rcv;
1734 SOCKBUF_LOCK(sb);
1735
1736 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
1737 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
1738 __func__, tid, len);
1739 m_freem(m);
1740 SOCKBUF_UNLOCK(sb);
1741 if (ulp_mode(toep) == ULP_MODE_TCPDDP)
1742 DDP_UNLOCK(toep);
1743 INP_WUNLOCK(inp);
1744
1745 CURVNET_SET(toep->vnet);
1746 NET_EPOCH_ENTER(et);
1747 INP_WLOCK(inp);
1748 tp = tcp_drop(tp, ECONNRESET);
1749 if (tp)
1750 INP_WUNLOCK(inp);
1751 NET_EPOCH_EXIT(et);
1752 CURVNET_RESTORE();
1753
1754 return (0);
1755 }
1756
1757 /* receive buffer autosize */
1758 MPASS(toep->vnet == so->so_vnet);
1759 CURVNET_SET(toep->vnet);
1760 if (sb->sb_flags & SB_AUTOSIZE &&
1761 V_tcp_do_autorcvbuf &&
1762 sb->sb_hiwat < V_tcp_autorcvbuf_max &&
1763 len > (sbspace(sb) / 8 * 7)) {
1764 unsigned int hiwat = sb->sb_hiwat;
1765 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
1766 V_tcp_autorcvbuf_max);
1767
1768 if (!sbreserve_locked(sb, newsize, so, NULL))
1769 sb->sb_flags &= ~SB_AUTOSIZE;
1770 }
1771
1772 if (ulp_mode(toep) == ULP_MODE_TCPDDP) {
1773 int changed = !(toep->ddp.flags & DDP_ON) ^ cpl->ddp_off;
1774
1775 if (toep->ddp.waiting_count != 0 || toep->ddp.active_count != 0)
1776 CTR3(KTR_CXGBE, "%s: tid %u, non-ddp rx (%d bytes)",
1777 __func__, tid, len);
1778
1779 if (changed) {
1780 if (toep->ddp.flags & DDP_SC_REQ)
1781 toep->ddp.flags ^= DDP_ON | DDP_SC_REQ;
1782 else {
1783 KASSERT(cpl->ddp_off == 1,
1784 ("%s: DDP switched on by itself.",
1785 __func__));
1786
1787 /* Fell out of DDP mode */
1788 toep->ddp.flags &= ~DDP_ON;
1789 CTR1(KTR_CXGBE, "%s: fell out of DDP mode",
1790 __func__);
1791
1792 insert_ddp_data(toep, ddp_placed);
1793 }
1794 }
1795
1796 if (toep->ddp.flags & DDP_ON) {
1797 /*
1798 * CPL_RX_DATA with DDP on can only be an indicate.
1799 * Start posting queued AIO requests via DDP. The
1800 * payload that arrived in this indicate is appended
1801 * to the socket buffer as usual.
1802 */
1803 handle_ddp_indicate(toep);
1804 }
1805 }
1806
1807 sbappendstream_locked(sb, m, 0);
1808 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
1809 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) {
1810 rx_credits = send_rx_credits(sc, toep, rx_credits);
1811 tp->rcv_wnd += rx_credits;
1812 tp->rcv_adv += rx_credits;
1813 }
1814
1815 if (ulp_mode(toep) == ULP_MODE_TCPDDP && toep->ddp.waiting_count > 0 &&
1816 sbavail(sb) != 0) {
1817 CTR2(KTR_CXGBE, "%s: tid %u queueing AIO task", __func__,
1818 tid);
1819 ddp_queue_toep(toep);
1820 }
1821 sorwakeup_locked(so);
1822 SOCKBUF_UNLOCK_ASSERT(sb);
1823 if (ulp_mode(toep) == ULP_MODE_TCPDDP)
1824 DDP_UNLOCK(toep);
1825
1826 INP_WUNLOCK(inp);
1827 CURVNET_RESTORE();
1828 return (0);
1829 }
1830
1831 static int
do_fw4_ack(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1832 do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1833 {
1834 struct adapter *sc = iq->adapter;
1835 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
1836 unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
1837 struct toepcb *toep = lookup_tid(sc, tid);
1838 struct inpcb *inp;
1839 struct tcpcb *tp;
1840 struct socket *so;
1841 uint8_t credits = cpl->credits;
1842 struct ofld_tx_sdesc *txsd;
1843 int plen;
1844 #ifdef INVARIANTS
1845 unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl)));
1846 #endif
1847
1848 /*
1849 * Very unusual case: we'd sent a flowc + abort_req for a synq entry and
1850 * now this comes back carrying the credits for the flowc.
1851 */
1852 if (__predict_false(toep->flags & TPF_SYNQE)) {
1853 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1854 ("%s: credits for a synq entry %p", __func__, toep));
1855 return (0);
1856 }
1857
1858 inp = toep->inp;
1859
1860 KASSERT(opcode == CPL_FW4_ACK,
1861 ("%s: unexpected opcode 0x%x", __func__, opcode));
1862 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1863 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1864
1865 INP_WLOCK(inp);
1866
1867 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) {
1868 INP_WUNLOCK(inp);
1869 return (0);
1870 }
1871
1872 KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0,
1873 ("%s: inp_flags 0x%x", __func__, inp->inp_flags));
1874
1875 tp = intotcpcb(inp);
1876
1877 if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) {
1878 tcp_seq snd_una = be32toh(cpl->snd_una);
1879
1880 #ifdef INVARIANTS
1881 if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) {
1882 log(LOG_ERR,
1883 "%s: unexpected seq# %x for TID %u, snd_una %x\n",
1884 __func__, snd_una, toep->tid, tp->snd_una);
1885 }
1886 #endif
1887
1888 if (tp->snd_una != snd_una) {
1889 tp->snd_una = snd_una;
1890 tp->ts_recent_age = tcp_ts_getticks();
1891 }
1892 }
1893
1894 #ifdef VERBOSE_TRACES
1895 CTR3(KTR_CXGBE, "%s: tid %d credits %u", __func__, tid, credits);
1896 #endif
1897 so = inp->inp_socket;
1898 txsd = &toep->txsd[toep->txsd_cidx];
1899 plen = 0;
1900 while (credits) {
1901 KASSERT(credits >= txsd->tx_credits,
1902 ("%s: too many (or partial) credits", __func__));
1903 credits -= txsd->tx_credits;
1904 toep->tx_credits += txsd->tx_credits;
1905 plen += txsd->plen;
1906 if (txsd->iv_buffer) {
1907 free(txsd->iv_buffer, M_CXGBE);
1908 txsd->iv_buffer = NULL;
1909 }
1910 txsd++;
1911 toep->txsd_avail++;
1912 KASSERT(toep->txsd_avail <= toep->txsd_total,
1913 ("%s: txsd avail > total", __func__));
1914 if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) {
1915 txsd = &toep->txsd[0];
1916 toep->txsd_cidx = 0;
1917 }
1918 }
1919
1920 if (toep->tx_credits == toep->tx_total) {
1921 toep->tx_nocompl = 0;
1922 toep->plen_nocompl = 0;
1923 }
1924
1925 if (toep->flags & TPF_TX_SUSPENDED &&
1926 toep->tx_credits >= toep->tx_total / 4) {
1927 #ifdef VERBOSE_TRACES
1928 CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__,
1929 tid);
1930 #endif
1931 toep->flags &= ~TPF_TX_SUSPENDED;
1932 CURVNET_SET(toep->vnet);
1933 t4_push_data(sc, toep, plen);
1934 CURVNET_RESTORE();
1935 } else if (plen > 0) {
1936 struct sockbuf *sb = &so->so_snd;
1937 int sbu;
1938
1939 SOCKBUF_LOCK(sb);
1940 sbu = sbused(sb);
1941 if (ulp_mode(toep) == ULP_MODE_ISCSI) {
1942 if (__predict_false(sbu > 0)) {
1943 /*
1944 * The data transmitted before the
1945 * tid's ULP mode changed to ISCSI is
1946 * still in so_snd. Incoming credits
1947 * should account for so_snd first.
1948 */
1949 sbdrop_locked(sb, min(sbu, plen));
1950 plen -= min(sbu, plen);
1951 }
1952 sowwakeup_locked(so); /* unlocks so_snd */
1953 rqdrop_locked(&toep->ulp_pdu_reclaimq, plen);
1954 } else {
1955 #ifdef VERBOSE_TRACES
1956 CTR3(KTR_CXGBE, "%s: tid %d dropped %d bytes", __func__,
1957 tid, plen);
1958 #endif
1959 sbdrop_locked(sb, plen);
1960 if (tls_tx_key(toep) &&
1961 toep->tls.mode == TLS_MODE_TLSOM) {
1962 struct tls_ofld_info *tls_ofld = &toep->tls;
1963
1964 MPASS(tls_ofld->sb_off >= plen);
1965 tls_ofld->sb_off -= plen;
1966 }
1967 if (!TAILQ_EMPTY(&toep->aiotx_jobq))
1968 t4_aiotx_queue_toep(so, toep);
1969 sowwakeup_locked(so); /* unlocks so_snd */
1970 }
1971 SOCKBUF_UNLOCK_ASSERT(sb);
1972 }
1973
1974 INP_WUNLOCK(inp);
1975
1976 return (0);
1977 }
1978
1979 void
t4_set_tcb_field(struct adapter * sc,struct sge_wrq * wrq,struct toepcb * toep,uint16_t word,uint64_t mask,uint64_t val,int reply,int cookie)1980 t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep,
1981 uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie)
1982 {
1983 struct wrqe *wr;
1984 struct cpl_set_tcb_field *req;
1985 struct ofld_tx_sdesc *txsd;
1986
1987 MPASS((cookie & ~M_COOKIE) == 0);
1988 if (reply) {
1989 MPASS(cookie != CPL_COOKIE_RESERVED);
1990 }
1991
1992 wr = alloc_wrqe(sizeof(*req), wrq);
1993 if (wr == NULL) {
1994 /* XXX */
1995 panic("%s: allocation failure.", __func__);
1996 }
1997 req = wrtod(wr);
1998
1999 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid);
2000 req->reply_ctrl = htobe16(V_QUEUENO(toep->ofld_rxq->iq.abs_id));
2001 if (reply == 0)
2002 req->reply_ctrl |= htobe16(F_NO_REPLY);
2003 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie));
2004 req->mask = htobe64(mask);
2005 req->val = htobe64(val);
2006 if (wrq->eq.type == EQ_OFLD) {
2007 txsd = &toep->txsd[toep->txsd_pidx];
2008 txsd->tx_credits = howmany(sizeof(*req), 16);
2009 txsd->plen = 0;
2010 KASSERT(toep->tx_credits >= txsd->tx_credits &&
2011 toep->txsd_avail > 0,
2012 ("%s: not enough credits (%d)", __func__,
2013 toep->tx_credits));
2014 toep->tx_credits -= txsd->tx_credits;
2015 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
2016 toep->txsd_pidx = 0;
2017 toep->txsd_avail--;
2018 }
2019
2020 t4_wrq_tx(sc, wr);
2021 }
2022
2023 void
t4_init_cpl_io_handlers(void)2024 t4_init_cpl_io_handlers(void)
2025 {
2026
2027 t4_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close);
2028 t4_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl);
2029 t4_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req);
2030 t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl,
2031 CPL_COOKIE_TOM);
2032 t4_register_cpl_handler(CPL_RX_DATA, do_rx_data);
2033 t4_register_shared_cpl_handler(CPL_FW4_ACK, do_fw4_ack, CPL_COOKIE_TOM);
2034 }
2035
2036 void
t4_uninit_cpl_io_handlers(void)2037 t4_uninit_cpl_io_handlers(void)
2038 {
2039
2040 t4_register_cpl_handler(CPL_PEER_CLOSE, NULL);
2041 t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL);
2042 t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL);
2043 t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, NULL, CPL_COOKIE_TOM);
2044 t4_register_cpl_handler(CPL_RX_DATA, NULL);
2045 t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM);
2046 }
2047
2048 /*
2049 * Use the 'backend1' field in AIO jobs to hold an error that should
2050 * be reported when the job is completed, the 'backend3' field to
2051 * store the amount of data sent by the AIO job so far, and the
2052 * 'backend4' field to hold a reference count on the job.
2053 *
2054 * Each unmapped mbuf holds a reference on the job as does the queue
2055 * so long as the job is queued.
2056 */
2057 #define aio_error backend1
2058 #define aio_sent backend3
2059 #define aio_refs backend4
2060
2061 #define jobtotid(job) \
2062 (((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid)
2063
2064 static void
aiotx_free_job(struct kaiocb * job)2065 aiotx_free_job(struct kaiocb *job)
2066 {
2067 long status;
2068 int error;
2069
2070 if (refcount_release(&job->aio_refs) == 0)
2071 return;
2072
2073 error = (intptr_t)job->aio_error;
2074 status = job->aio_sent;
2075 #ifdef VERBOSE_TRACES
2076 CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__,
2077 jobtotid(job), job, status, error);
2078 #endif
2079 if (error != 0 && status != 0)
2080 error = 0;
2081 if (error == ECANCELED)
2082 aio_cancel(job);
2083 else if (error)
2084 aio_complete(job, -1, error);
2085 else {
2086 job->msgsnd = 1;
2087 aio_complete(job, status, 0);
2088 }
2089 }
2090
2091 static void
aiotx_free_pgs(struct mbuf * m)2092 aiotx_free_pgs(struct mbuf *m)
2093 {
2094 struct kaiocb *job;
2095 vm_page_t pg;
2096
2097 M_ASSERTEXTPG(m);
2098 job = m->m_ext.ext_arg1;
2099 #ifdef VERBOSE_TRACES
2100 CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__,
2101 m->m_len, jobtotid(job));
2102 #endif
2103
2104 for (int i = 0; i < m->m_epg_npgs; i++) {
2105 pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
2106 vm_page_unwire(pg, PQ_ACTIVE);
2107 }
2108
2109 aiotx_free_job(job);
2110 }
2111
2112 /*
2113 * Allocate a chain of unmapped mbufs describing the next 'len' bytes
2114 * of an AIO job.
2115 */
2116 static struct mbuf *
alloc_aiotx_mbuf(struct kaiocb * job,int len)2117 alloc_aiotx_mbuf(struct kaiocb *job, int len)
2118 {
2119 struct vmspace *vm;
2120 vm_page_t pgs[MBUF_PEXT_MAX_PGS];
2121 struct mbuf *m, *top, *last;
2122 vm_map_t map;
2123 vm_offset_t start;
2124 int i, mlen, npages, pgoff;
2125
2126 KASSERT(job->aio_sent + len <= job->uaiocb.aio_nbytes,
2127 ("%s(%p, %d): request to send beyond end of buffer", __func__,
2128 job, len));
2129
2130 /*
2131 * The AIO subsystem will cancel and drain all requests before
2132 * permitting a process to exit or exec, so p_vmspace should
2133 * be stable here.
2134 */
2135 vm = job->userproc->p_vmspace;
2136 map = &vm->vm_map;
2137 start = (uintptr_t)job->uaiocb.aio_buf + job->aio_sent;
2138 pgoff = start & PAGE_MASK;
2139
2140 top = NULL;
2141 last = NULL;
2142 while (len > 0) {
2143 mlen = imin(len, MBUF_PEXT_MAX_PGS * PAGE_SIZE - pgoff);
2144 KASSERT(mlen == len || ((start + mlen) & PAGE_MASK) == 0,
2145 ("%s: next start (%#jx + %#x) is not page aligned",
2146 __func__, (uintmax_t)start, mlen));
2147
2148 npages = vm_fault_quick_hold_pages(map, start, mlen,
2149 VM_PROT_WRITE, pgs, nitems(pgs));
2150 if (npages < 0)
2151 break;
2152
2153 m = mb_alloc_ext_pgs(M_WAITOK, aiotx_free_pgs);
2154 if (m == NULL) {
2155 vm_page_unhold_pages(pgs, npages);
2156 break;
2157 }
2158
2159 m->m_epg_1st_off = pgoff;
2160 m->m_epg_npgs = npages;
2161 if (npages == 1) {
2162 KASSERT(mlen + pgoff <= PAGE_SIZE,
2163 ("%s: single page is too large (off %d len %d)",
2164 __func__, pgoff, mlen));
2165 m->m_epg_last_len = mlen;
2166 } else {
2167 m->m_epg_last_len = mlen - (PAGE_SIZE - pgoff) -
2168 (npages - 2) * PAGE_SIZE;
2169 }
2170 for (i = 0; i < npages; i++)
2171 m->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
2172
2173 m->m_len = mlen;
2174 m->m_ext.ext_size = npages * PAGE_SIZE;
2175 m->m_ext.ext_arg1 = job;
2176 refcount_acquire(&job->aio_refs);
2177
2178 #ifdef VERBOSE_TRACES
2179 CTR5(KTR_CXGBE, "%s: tid %d, new mbuf %p for job %p, npages %d",
2180 __func__, jobtotid(job), m, job, npages);
2181 #endif
2182
2183 if (top == NULL)
2184 top = m;
2185 else
2186 last->m_next = m;
2187 last = m;
2188
2189 len -= mlen;
2190 start += mlen;
2191 pgoff = 0;
2192 }
2193
2194 return (top);
2195 }
2196
2197 static void
t4_aiotx_process_job(struct toepcb * toep,struct socket * so,struct kaiocb * job)2198 t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job)
2199 {
2200 struct sockbuf *sb;
2201 struct file *fp;
2202 struct inpcb *inp;
2203 struct tcpcb *tp;
2204 struct mbuf *m;
2205 int error, len;
2206 bool moretocome, sendmore;
2207
2208 sb = &so->so_snd;
2209 SOCKBUF_UNLOCK(sb);
2210 fp = job->fd_file;
2211 m = NULL;
2212
2213 #ifdef MAC
2214 error = mac_socket_check_send(fp->f_cred, so);
2215 if (error != 0)
2216 goto out;
2217 #endif
2218
2219 /* Inline sosend_generic(). */
2220
2221 error = SOCK_IO_SEND_LOCK(so, SBL_WAIT);
2222 MPASS(error == 0);
2223
2224 sendanother:
2225 SOCKBUF_LOCK(sb);
2226 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
2227 SOCKBUF_UNLOCK(sb);
2228 SOCK_IO_SEND_UNLOCK(so);
2229 if ((so->so_options & SO_NOSIGPIPE) == 0) {
2230 PROC_LOCK(job->userproc);
2231 kern_psignal(job->userproc, SIGPIPE);
2232 PROC_UNLOCK(job->userproc);
2233 }
2234 error = EPIPE;
2235 goto out;
2236 }
2237 if (so->so_error) {
2238 error = so->so_error;
2239 so->so_error = 0;
2240 SOCKBUF_UNLOCK(sb);
2241 SOCK_IO_SEND_UNLOCK(so);
2242 goto out;
2243 }
2244 if ((so->so_state & SS_ISCONNECTED) == 0) {
2245 SOCKBUF_UNLOCK(sb);
2246 SOCK_IO_SEND_UNLOCK(so);
2247 error = ENOTCONN;
2248 goto out;
2249 }
2250 if (sbspace(sb) < sb->sb_lowat) {
2251 MPASS(job->aio_sent == 0 || !(so->so_state & SS_NBIO));
2252
2253 /*
2254 * Don't block if there is too little room in the socket
2255 * buffer. Instead, requeue the request.
2256 */
2257 if (!aio_set_cancel_function(job, t4_aiotx_cancel)) {
2258 SOCKBUF_UNLOCK(sb);
2259 SOCK_IO_SEND_UNLOCK(so);
2260 error = ECANCELED;
2261 goto out;
2262 }
2263 TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list);
2264 SOCKBUF_UNLOCK(sb);
2265 SOCK_IO_SEND_UNLOCK(so);
2266 goto out;
2267 }
2268
2269 /*
2270 * Write as much data as the socket permits, but no more than a
2271 * a single sndbuf at a time.
2272 */
2273 len = sbspace(sb);
2274 if (len > job->uaiocb.aio_nbytes - job->aio_sent) {
2275 len = job->uaiocb.aio_nbytes - job->aio_sent;
2276 moretocome = false;
2277 } else
2278 moretocome = true;
2279 if (len > toep->params.sndbuf) {
2280 len = toep->params.sndbuf;
2281 sendmore = true;
2282 } else
2283 sendmore = false;
2284
2285 if (!TAILQ_EMPTY(&toep->aiotx_jobq))
2286 moretocome = true;
2287 SOCKBUF_UNLOCK(sb);
2288 MPASS(len != 0);
2289
2290 m = alloc_aiotx_mbuf(job, len);
2291 if (m == NULL) {
2292 SOCK_IO_SEND_UNLOCK(so);
2293 error = EFAULT;
2294 goto out;
2295 }
2296
2297 /* Inlined tcp_usr_send(). */
2298
2299 inp = toep->inp;
2300 INP_WLOCK(inp);
2301 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
2302 INP_WUNLOCK(inp);
2303 SOCK_IO_SEND_UNLOCK(so);
2304 error = ECONNRESET;
2305 goto out;
2306 }
2307
2308 job->aio_sent += m_length(m, NULL);
2309
2310 sbappendstream(sb, m, 0);
2311 m = NULL;
2312
2313 if (!(inp->inp_flags & INP_DROPPED)) {
2314 tp = intotcpcb(inp);
2315 if (moretocome)
2316 tp->t_flags |= TF_MORETOCOME;
2317 error = tp->t_fb->tfb_tcp_output(tp);
2318 if (moretocome)
2319 tp->t_flags &= ~TF_MORETOCOME;
2320 }
2321
2322 INP_WUNLOCK(inp);
2323 if (sendmore)
2324 goto sendanother;
2325 SOCK_IO_SEND_UNLOCK(so);
2326
2327 if (error)
2328 goto out;
2329
2330 /*
2331 * If this is a blocking socket and the request has not been
2332 * fully completed, requeue it until the socket is ready
2333 * again.
2334 */
2335 if (job->aio_sent < job->uaiocb.aio_nbytes &&
2336 !(so->so_state & SS_NBIO)) {
2337 SOCKBUF_LOCK(sb);
2338 if (!aio_set_cancel_function(job, t4_aiotx_cancel)) {
2339 SOCKBUF_UNLOCK(sb);
2340 error = ECANCELED;
2341 goto out;
2342 }
2343 TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list);
2344 return;
2345 }
2346
2347 /*
2348 * If the request will not be requeued, drop the queue's
2349 * reference to the job. Any mbufs in flight should still
2350 * hold a reference, but this drops the reference that the
2351 * queue owns while it is waiting to queue mbufs to the
2352 * socket.
2353 */
2354 aiotx_free_job(job);
2355
2356 out:
2357 if (error) {
2358 job->aio_error = (void *)(intptr_t)error;
2359 aiotx_free_job(job);
2360 }
2361 m_freem(m);
2362 SOCKBUF_LOCK(sb);
2363 }
2364
2365 static void
t4_aiotx_task(void * context,int pending)2366 t4_aiotx_task(void *context, int pending)
2367 {
2368 struct toepcb *toep = context;
2369 struct socket *so;
2370 struct kaiocb *job;
2371
2372 so = toep->aiotx_so;
2373 CURVNET_SET(toep->vnet);
2374 SOCKBUF_LOCK(&so->so_snd);
2375 while (!TAILQ_EMPTY(&toep->aiotx_jobq) && sowriteable(so)) {
2376 job = TAILQ_FIRST(&toep->aiotx_jobq);
2377 TAILQ_REMOVE(&toep->aiotx_jobq, job, list);
2378 if (!aio_clear_cancel_function(job))
2379 continue;
2380
2381 t4_aiotx_process_job(toep, so, job);
2382 }
2383 toep->aiotx_so = NULL;
2384 SOCKBUF_UNLOCK(&so->so_snd);
2385 CURVNET_RESTORE();
2386
2387 free_toepcb(toep);
2388 SOCK_LOCK(so);
2389 sorele(so);
2390 }
2391
2392 static void
t4_aiotx_queue_toep(struct socket * so,struct toepcb * toep)2393 t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep)
2394 {
2395
2396 SOCKBUF_LOCK_ASSERT(&toep->inp->inp_socket->so_snd);
2397 #ifdef VERBOSE_TRACES
2398 CTR3(KTR_CXGBE, "%s: queueing aiotx task for tid %d, active = %s",
2399 __func__, toep->tid, toep->aiotx_so != NULL ? "true" : "false");
2400 #endif
2401 if (toep->aiotx_so != NULL)
2402 return;
2403 soref(so);
2404 toep->aiotx_so = so;
2405 hold_toepcb(toep);
2406 soaio_enqueue(&toep->aiotx_task);
2407 }
2408
2409 static void
t4_aiotx_cancel(struct kaiocb * job)2410 t4_aiotx_cancel(struct kaiocb *job)
2411 {
2412 struct socket *so;
2413 struct sockbuf *sb;
2414 struct tcpcb *tp;
2415 struct toepcb *toep;
2416
2417 so = job->fd_file->f_data;
2418 tp = so_sototcpcb(so);
2419 toep = tp->t_toe;
2420 MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE);
2421 sb = &so->so_snd;
2422
2423 SOCKBUF_LOCK(sb);
2424 if (!aio_cancel_cleared(job))
2425 TAILQ_REMOVE(&toep->aiotx_jobq, job, list);
2426 SOCKBUF_UNLOCK(sb);
2427
2428 job->aio_error = (void *)(intptr_t)ECANCELED;
2429 aiotx_free_job(job);
2430 }
2431
2432 int
t4_aio_queue_aiotx(struct socket * so,struct kaiocb * job)2433 t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job)
2434 {
2435 struct tcpcb *tp = so_sototcpcb(so);
2436 struct toepcb *toep = tp->t_toe;
2437 struct adapter *sc = td_adapter(toep->td);
2438
2439 /* This only handles writes. */
2440 if (job->uaiocb.aio_lio_opcode != LIO_WRITE)
2441 return (EOPNOTSUPP);
2442
2443 if (!sc->tt.tx_zcopy)
2444 return (EOPNOTSUPP);
2445
2446 if (tls_tx_key(toep))
2447 return (EOPNOTSUPP);
2448
2449 SOCKBUF_LOCK(&so->so_snd);
2450 #ifdef VERBOSE_TRACES
2451 CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid);
2452 #endif
2453 if (!aio_set_cancel_function(job, t4_aiotx_cancel))
2454 panic("new job was cancelled");
2455 refcount_init(&job->aio_refs, 1);
2456 TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list);
2457 if (sowriteable(so))
2458 t4_aiotx_queue_toep(so, toep);
2459 SOCKBUF_UNLOCK(&so->so_snd);
2460 return (0);
2461 }
2462
2463 void
aiotx_init_toep(struct toepcb * toep)2464 aiotx_init_toep(struct toepcb *toep)
2465 {
2466
2467 TAILQ_INIT(&toep->aiotx_jobq);
2468 TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep);
2469 }
2470 #endif
2471