1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2012 The FreeBSD Foundation
5 *
6 * This software was developed by Edward Tomasz Napierala under sponsorship
7 * from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 /*
32 * CTL frontend for the iSCSI protocol.
33 */
34
35 #include <sys/cdefs.h>
36 #include <sys/param.h>
37 #include <sys/capsicum.h>
38 #include <sys/condvar.h>
39 #include <sys/endian.h>
40 #include <sys/file.h>
41 #include <sys/kernel.h>
42 #include <sys/kthread.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/mutex.h>
47 #include <sys/queue.h>
48 #include <sys/sbuf.h>
49 #include <sys/socket.h>
50 #include <sys/sysctl.h>
51 #include <sys/systm.h>
52 #include <sys/uio.h>
53 #include <sys/unistd.h>
54 #include <sys/nv.h>
55 #include <sys/dnv.h>
56 #include <vm/uma.h>
57
58 #include <cam/scsi/scsi_all.h>
59 #include <cam/scsi/scsi_da.h>
60 #include <cam/ctl/ctl_io.h>
61 #include <cam/ctl/ctl.h>
62 #include <cam/ctl/ctl_backend.h>
63 #include <cam/ctl/ctl_error.h>
64 #include <cam/ctl/ctl_frontend.h>
65 #include <cam/ctl/ctl_debug.h>
66 #include <cam/ctl/ctl_ha.h>
67 #include <cam/ctl/ctl_ioctl.h>
68 #include <cam/ctl/ctl_private.h>
69
70 #include <dev/iscsi/icl.h>
71 #include <dev/iscsi/icl_wrappers.h>
72 #include <dev/iscsi/iscsi_proto.h>
73 #include <cam/ctl/ctl_frontend_iscsi.h>
74
75 #ifdef ICL_KERNEL_PROXY
76 #include <sys/socketvar.h>
77 #endif
78
79 #ifdef ICL_KERNEL_PROXY
80 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
81 #endif
82
83 /* Used for internal nexus reset task. */
84 #define ISCSI_BHS_OPCODE_INTERNAL 0x3e
85
86 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
87 static uma_zone_t cfiscsi_data_wait_zone;
88
89 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
90 "CAM Target Layer iSCSI Frontend");
91 static int debug = 1;
92 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
93 &debug, 1, "Enable debug messages");
94 static int ping_timeout = 5;
95 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
96 &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
97 static int login_timeout = 60;
98 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
99 &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
100 static int maxtags = 256;
101 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN,
102 &maxtags, 0, "Max number of requests queued by initiator");
103
104 #define CFISCSI_DEBUG(X, ...) \
105 do { \
106 if (debug > 1) { \
107 printf("%s: " X "\n", \
108 __func__, ## __VA_ARGS__); \
109 } \
110 } while (0)
111
112 #define CFISCSI_WARN(X, ...) \
113 do { \
114 if (debug > 0) { \
115 printf("WARNING: %s: " X "\n", \
116 __func__, ## __VA_ARGS__); \
117 } \
118 } while (0)
119
120 #define CFISCSI_SESSION_DEBUG(S, X, ...) \
121 do { \
122 if (debug > 1) { \
123 printf("%s: %s (%s): " X "\n", \
124 __func__, S->cs_initiator_addr, \
125 S->cs_initiator_name, ## __VA_ARGS__); \
126 } \
127 } while (0)
128
129 #define CFISCSI_SESSION_WARN(S, X, ...) \
130 do { \
131 if (debug > 0) { \
132 printf("WARNING: %s (%s): " X "\n", \
133 S->cs_initiator_addr, \
134 S->cs_initiator_name, ## __VA_ARGS__); \
135 } \
136 } while (0)
137
138 #define CFISCSI_SESSION_LOCK(X) mtx_lock(&X->cs_lock)
139 #define CFISCSI_SESSION_UNLOCK(X) mtx_unlock(&X->cs_lock)
140 #define CFISCSI_SESSION_LOCK_ASSERT(X) mtx_assert(&X->cs_lock, MA_OWNED)
141
142 #define CONN_SESSION(X) ((struct cfiscsi_session *)(X)->ic_prv0)
143 #define PDU_SESSION(X) CONN_SESSION((X)->ip_conn)
144
145 struct cfiscsi_priv {
146 void *request;
147 uint32_t expdatasn;
148 uint32_t r2tsn;
149 };
150 #define PRIV(io) \
151 ((struct cfiscsi_priv *)&(io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND])
152 #define PRIV_REQUEST(io) PRIV(io)->request
153 #define PRIV_EXPDATASN(io) PRIV(io)->expdatasn
154 #define PRIV_R2TSN(io) PRIV(io)->r2tsn
155
156 static int cfiscsi_init(void);
157 static int cfiscsi_shutdown(void);
158 static void cfiscsi_online(void *arg);
159 static void cfiscsi_offline(void *arg);
160 static int cfiscsi_info(void *arg, struct sbuf *sb);
161 static int cfiscsi_ioctl(struct cdev *dev,
162 u_long cmd, caddr_t addr, int flag, struct thread *td);
163 static void cfiscsi_datamove(union ctl_io *io);
164 static void cfiscsi_datamove_in(union ctl_io *io);
165 static void cfiscsi_datamove_out(union ctl_io *io);
166 static void cfiscsi_done(union ctl_io *io);
167 static bool cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
168 static void cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
169 static void cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
170 static void cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
171 static void cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
172 static void cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
173 static void cfiscsi_session_terminate(struct cfiscsi_session *cs);
174 static struct cfiscsi_data_wait *cfiscsi_data_wait_new(
175 struct cfiscsi_session *cs, union ctl_io *io,
176 uint32_t initiator_task_tag,
177 uint32_t *target_transfer_tagp);
178 static void cfiscsi_data_wait_free(struct cfiscsi_session *cs,
179 struct cfiscsi_data_wait *cdw);
180 static struct cfiscsi_target *cfiscsi_target_find(struct cfiscsi_softc
181 *softc, const char *name, uint16_t tag);
182 static struct cfiscsi_target *cfiscsi_target_find_or_create(
183 struct cfiscsi_softc *softc, const char *name, const char *alias,
184 uint16_t tag);
185 static void cfiscsi_target_release(struct cfiscsi_target *ct);
186 static void cfiscsi_session_delete(struct cfiscsi_session *cs);
187
188 static struct cfiscsi_softc cfiscsi_softc;
189
190 static struct ctl_frontend cfiscsi_frontend =
191 {
192 .name = "iscsi",
193 .init = cfiscsi_init,
194 .ioctl = cfiscsi_ioctl,
195 .shutdown = cfiscsi_shutdown,
196 };
197 CTL_FRONTEND_DECLARE(cfiscsi, cfiscsi_frontend);
198 MODULE_DEPEND(cfiscsi, icl, 1, 1, 1);
199
200 static struct icl_pdu *
cfiscsi_pdu_new_response(struct icl_pdu * request,int flags)201 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
202 {
203
204 return (icl_pdu_new(request->ip_conn, flags));
205 }
206
207 static bool
cfiscsi_pdu_update_cmdsn(const struct icl_pdu * request)208 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
209 {
210 const struct iscsi_bhs_scsi_command *bhssc;
211 struct cfiscsi_session *cs;
212 uint32_t cmdsn, curcmdsn;
213
214 cs = PDU_SESSION(request);
215
216 /*
217 * Every incoming PDU - not just NOP-Out - resets the ping timer.
218 * The purpose of the timeout is to reset the connection when it stalls;
219 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
220 * in some queue.
221 */
222 cs->cs_timeout = 0;
223
224 /*
225 * Immediate commands carry cmdsn, but it is neither incremented nor
226 * verified.
227 */
228 if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE)
229 return (false);
230
231 /*
232 * Data-Out PDUs don't contain CmdSN.
233 */
234 if (request->ip_bhs->bhs_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
235 return (false);
236
237 /*
238 * We're only using fields common for all the request
239 * (initiator -> target) PDUs.
240 */
241 bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
242 curcmdsn = cmdsn = ntohl(bhssc->bhssc_cmdsn);
243
244 /*
245 * Increment session cmdsn and exit if we received the expected value.
246 */
247 do {
248 if (atomic_fcmpset_32(&cs->cs_cmdsn, &curcmdsn, cmdsn + 1))
249 return (false);
250 } while (curcmdsn == cmdsn);
251
252 /*
253 * The target MUST silently ignore any non-immediate command outside
254 * of this range.
255 */
256 if (ISCSI_SNLT(cmdsn, curcmdsn) ||
257 ISCSI_SNGT(cmdsn, curcmdsn - 1 + maxtags)) {
258 CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
259 "while expected %u", cmdsn, curcmdsn);
260 return (true);
261 }
262
263 /*
264 * We don't support multiple connections now, so any discontinuity in
265 * CmdSN means lost PDUs. Since we don't support PDU retransmission --
266 * terminate the connection.
267 */
268 CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
269 "while expected %u; dropping connection",
270 cmdsn, curcmdsn);
271 cfiscsi_session_terminate(cs);
272 return (true);
273 }
274
275 static void
cfiscsi_pdu_handle(struct icl_pdu * request)276 cfiscsi_pdu_handle(struct icl_pdu *request)
277 {
278 struct cfiscsi_session *cs;
279 bool ignore;
280
281 cs = PDU_SESSION(request);
282
283 ignore = cfiscsi_pdu_update_cmdsn(request);
284 if (ignore) {
285 icl_pdu_free(request);
286 return;
287 }
288
289 /*
290 * Handle the PDU; this includes e.g. receiving the remaining
291 * part of PDU and submitting the SCSI command to CTL
292 * or queueing a reply. The handling routine is responsible
293 * for freeing the PDU when it's no longer needed.
294 */
295 switch (request->ip_bhs->bhs_opcode &
296 ~ISCSI_BHS_OPCODE_IMMEDIATE) {
297 case ISCSI_BHS_OPCODE_NOP_OUT:
298 cfiscsi_pdu_handle_nop_out(request);
299 break;
300 case ISCSI_BHS_OPCODE_SCSI_COMMAND:
301 cfiscsi_pdu_handle_scsi_command(request);
302 break;
303 case ISCSI_BHS_OPCODE_TASK_REQUEST:
304 cfiscsi_pdu_handle_task_request(request);
305 break;
306 case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
307 cfiscsi_pdu_handle_data_out(request);
308 break;
309 case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
310 cfiscsi_pdu_handle_logout_request(request);
311 break;
312 default:
313 CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
314 "opcode 0x%x; dropping connection",
315 request->ip_bhs->bhs_opcode);
316 icl_pdu_free(request);
317 cfiscsi_session_terminate(cs);
318 }
319
320 }
321
322 static void
cfiscsi_receive_callback(struct icl_pdu * request)323 cfiscsi_receive_callback(struct icl_pdu *request)
324 {
325 #ifdef ICL_KERNEL_PROXY
326 struct cfiscsi_session *cs;
327
328 cs = PDU_SESSION(request);
329 if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
330 if (cs->cs_login_pdu == NULL)
331 cs->cs_login_pdu = request;
332 else
333 icl_pdu_free(request);
334 cv_signal(&cs->cs_login_cv);
335 return;
336 }
337 #endif
338
339 cfiscsi_pdu_handle(request);
340 }
341
342 static void
cfiscsi_error_callback(struct icl_conn * ic)343 cfiscsi_error_callback(struct icl_conn *ic)
344 {
345 struct cfiscsi_session *cs;
346
347 cs = CONN_SESSION(ic);
348
349 CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
350 cfiscsi_session_terminate(cs);
351 }
352
353 static int
cfiscsi_pdu_prepare(struct icl_pdu * response)354 cfiscsi_pdu_prepare(struct icl_pdu *response)
355 {
356 struct cfiscsi_session *cs;
357 struct iscsi_bhs_scsi_response *bhssr;
358 bool advance_statsn = true;
359 uint32_t cmdsn;
360
361 cs = PDU_SESSION(response);
362
363 CFISCSI_SESSION_LOCK_ASSERT(cs);
364
365 /*
366 * We're only using fields common for all the response
367 * (target -> initiator) PDUs.
368 */
369 bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
370
371 /*
372 * 10.8.3: "The StatSN for this connection is not advanced
373 * after this PDU is sent."
374 */
375 if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
376 advance_statsn = false;
377
378 /*
379 * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
380 * StatSN for the connection is not advanced after this PDU is sent."
381 */
382 if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
383 bhssr->bhssr_initiator_task_tag == 0xffffffff)
384 advance_statsn = false;
385
386 /*
387 * See the comment below - StatSN is not meaningful and must
388 * not be advanced.
389 */
390 if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
391 (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
392 advance_statsn = false;
393
394 /*
395 * 10.7.3: "The fields StatSN, Status, and Residual Count
396 * only have meaningful content if the S bit is set to 1."
397 */
398 if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
399 (bhssr->bhssr_flags & BHSDI_FLAGS_S))
400 bhssr->bhssr_statsn = htonl(cs->cs_statsn);
401 cmdsn = cs->cs_cmdsn;
402 bhssr->bhssr_expcmdsn = htonl(cmdsn);
403 bhssr->bhssr_maxcmdsn = htonl(cmdsn - 1 +
404 imax(0, maxtags - cs->cs_outstanding_ctl_pdus));
405
406 if (advance_statsn)
407 cs->cs_statsn++;
408
409 return (0);
410 }
411
412 static void
cfiscsi_pdu_queue(struct icl_pdu * response)413 cfiscsi_pdu_queue(struct icl_pdu *response)
414 {
415 struct cfiscsi_session *cs;
416
417 cs = PDU_SESSION(response);
418
419 CFISCSI_SESSION_LOCK(cs);
420 cfiscsi_pdu_prepare(response);
421 icl_pdu_queue(response);
422 CFISCSI_SESSION_UNLOCK(cs);
423 }
424
425 static void
cfiscsi_pdu_queue_cb(struct icl_pdu * response,icl_pdu_cb cb)426 cfiscsi_pdu_queue_cb(struct icl_pdu *response, icl_pdu_cb cb)
427 {
428 struct cfiscsi_session *cs = PDU_SESSION(response);
429
430 CFISCSI_SESSION_LOCK(cs);
431 cfiscsi_pdu_prepare(response);
432 icl_pdu_queue_cb(response, cb);
433 CFISCSI_SESSION_UNLOCK(cs);
434 }
435
436 static void
cfiscsi_pdu_handle_nop_out(struct icl_pdu * request)437 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
438 {
439 struct cfiscsi_session *cs;
440 struct iscsi_bhs_nop_out *bhsno;
441 struct iscsi_bhs_nop_in *bhsni;
442 struct icl_pdu *response;
443 void *data = NULL;
444 size_t datasize;
445 int error;
446
447 cs = PDU_SESSION(request);
448 bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
449
450 if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
451 /*
452 * Nothing to do, iscsi_pdu_update_statsn() already
453 * zeroed the timeout.
454 */
455 icl_pdu_free(request);
456 return;
457 }
458
459 datasize = icl_pdu_data_segment_length(request);
460 if (datasize > 0) {
461 data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
462 if (data == NULL) {
463 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
464 "dropping connection");
465 icl_pdu_free(request);
466 cfiscsi_session_terminate(cs);
467 return;
468 }
469 icl_pdu_get_data(request, 0, data, datasize);
470 }
471
472 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
473 if (response == NULL) {
474 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
475 "droppping connection");
476 free(data, M_CFISCSI);
477 icl_pdu_free(request);
478 cfiscsi_session_terminate(cs);
479 return;
480 }
481 bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
482 bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
483 bhsni->bhsni_flags = 0x80;
484 bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
485 bhsni->bhsni_target_transfer_tag = 0xffffffff;
486 if (datasize > 0) {
487 error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
488 if (error != 0) {
489 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
490 "dropping connection");
491 free(data, M_CFISCSI);
492 icl_pdu_free(request);
493 icl_pdu_free(response);
494 cfiscsi_session_terminate(cs);
495 return;
496 }
497 free(data, M_CFISCSI);
498 }
499
500 icl_pdu_free(request);
501 cfiscsi_pdu_queue(response);
502 }
503
504 static void
cfiscsi_pdu_handle_scsi_command(struct icl_pdu * request)505 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
506 {
507 struct iscsi_bhs_scsi_command *bhssc;
508 struct cfiscsi_session *cs;
509 union ctl_io *io;
510 int error;
511
512 cs = PDU_SESSION(request);
513 bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
514 //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
515 // bhssc->bhssc_initiator_task_tag);
516
517 if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
518 CFISCSI_SESSION_WARN(cs, "unsolicited data with "
519 "ImmediateData=No; dropping connection");
520 icl_pdu_free(request);
521 cfiscsi_session_terminate(cs);
522 return;
523 }
524 io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
525 ctl_zero_io(io);
526 PRIV_REQUEST(io) = request;
527 io->io_hdr.io_type = CTL_IO_SCSI;
528 io->io_hdr.nexus.initid = cs->cs_ctl_initid;
529 io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
530 io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhssc->bhssc_lun));
531 io->scsiio.priority = (bhssc->bhssc_pri & BHSSC_PRI_MASK) >>
532 BHSSC_PRI_SHIFT;
533 io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
534 switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
535 case BHSSC_FLAGS_ATTR_UNTAGGED:
536 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
537 break;
538 case BHSSC_FLAGS_ATTR_SIMPLE:
539 io->scsiio.tag_type = CTL_TAG_SIMPLE;
540 break;
541 case BHSSC_FLAGS_ATTR_ORDERED:
542 io->scsiio.tag_type = CTL_TAG_ORDERED;
543 break;
544 case BHSSC_FLAGS_ATTR_HOQ:
545 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
546 break;
547 case BHSSC_FLAGS_ATTR_ACA:
548 io->scsiio.tag_type = CTL_TAG_ACA;
549 break;
550 default:
551 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
552 CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
553 bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
554 break;
555 }
556 io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
557 memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
558 refcount_acquire(&cs->cs_outstanding_ctl_pdus);
559 error = ctl_run(io);
560 if (error != CTL_RETVAL_COMPLETE) {
561 CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d; "
562 "dropping connection", error);
563 ctl_free_io(io);
564 refcount_release(&cs->cs_outstanding_ctl_pdus);
565 icl_pdu_free(request);
566 cfiscsi_session_terminate(cs);
567 }
568 }
569
570 static void
cfiscsi_pdu_handle_task_request(struct icl_pdu * request)571 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
572 {
573 struct iscsi_bhs_task_management_request *bhstmr;
574 struct iscsi_bhs_task_management_response *bhstmr2;
575 struct icl_pdu *response;
576 struct cfiscsi_session *cs;
577 union ctl_io *io;
578 int error;
579
580 cs = PDU_SESSION(request);
581 bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
582 io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
583 ctl_zero_io(io);
584 PRIV_REQUEST(io) = request;
585 io->io_hdr.io_type = CTL_IO_TASK;
586 io->io_hdr.nexus.initid = cs->cs_ctl_initid;
587 io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
588 io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhstmr->bhstmr_lun));
589 io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
590
591 switch (bhstmr->bhstmr_function & ~0x80) {
592 case BHSTMR_FUNCTION_ABORT_TASK:
593 #if 0
594 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
595 #endif
596 io->taskio.task_action = CTL_TASK_ABORT_TASK;
597 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
598 break;
599 case BHSTMR_FUNCTION_ABORT_TASK_SET:
600 #if 0
601 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
602 #endif
603 io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
604 break;
605 case BHSTMR_FUNCTION_CLEAR_TASK_SET:
606 #if 0
607 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");
608 #endif
609 io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
610 break;
611 case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
612 #if 0
613 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
614 #endif
615 io->taskio.task_action = CTL_TASK_LUN_RESET;
616 break;
617 case BHSTMR_FUNCTION_TARGET_WARM_RESET:
618 #if 0
619 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
620 #endif
621 io->taskio.task_action = CTL_TASK_TARGET_RESET;
622 break;
623 case BHSTMR_FUNCTION_TARGET_COLD_RESET:
624 #if 0
625 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
626 #endif
627 io->taskio.task_action = CTL_TASK_TARGET_RESET;
628 break;
629 case BHSTMR_FUNCTION_QUERY_TASK:
630 #if 0
631 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");
632 #endif
633 io->taskio.task_action = CTL_TASK_QUERY_TASK;
634 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
635 break;
636 case BHSTMR_FUNCTION_QUERY_TASK_SET:
637 #if 0
638 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");
639 #endif
640 io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
641 break;
642 case BHSTMR_FUNCTION_I_T_NEXUS_RESET:
643 #if 0
644 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");
645 #endif
646 io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
647 break;
648 case BHSTMR_FUNCTION_QUERY_ASYNC_EVENT:
649 #if 0
650 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");
651 #endif
652 io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;
653 break;
654 default:
655 CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
656 bhstmr->bhstmr_function & ~0x80);
657 ctl_free_io(io);
658
659 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
660 if (response == NULL) {
661 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
662 "dropping connection");
663 icl_pdu_free(request);
664 cfiscsi_session_terminate(cs);
665 return;
666 }
667 bhstmr2 = (struct iscsi_bhs_task_management_response *)
668 response->ip_bhs;
669 bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
670 bhstmr2->bhstmr_flags = 0x80;
671 bhstmr2->bhstmr_response =
672 BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
673 bhstmr2->bhstmr_initiator_task_tag =
674 bhstmr->bhstmr_initiator_task_tag;
675 icl_pdu_free(request);
676 cfiscsi_pdu_queue(response);
677 return;
678 }
679
680 refcount_acquire(&cs->cs_outstanding_ctl_pdus);
681 error = ctl_run(io);
682 if (error != CTL_RETVAL_COMPLETE) {
683 CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d; "
684 "dropping connection", error);
685 ctl_free_io(io);
686 refcount_release(&cs->cs_outstanding_ctl_pdus);
687 icl_pdu_free(request);
688 cfiscsi_session_terminate(cs);
689 }
690 }
691
692 static bool
cfiscsi_handle_data_segment(struct icl_pdu * request,struct cfiscsi_data_wait * cdw)693 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
694 {
695 struct iscsi_bhs_data_out *bhsdo;
696 struct cfiscsi_session *cs;
697 struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
698 size_t copy_len, len, off, buffer_offset;
699 int ctl_sg_count;
700 union ctl_io *io;
701
702 cs = PDU_SESSION(request);
703
704 KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
705 ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
706 (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
707 ISCSI_BHS_OPCODE_SCSI_COMMAND,
708 ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
709
710 /*
711 * We're only using fields common for Data-Out and SCSI Command PDUs.
712 */
713 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
714
715 io = cdw->cdw_ctl_io;
716 KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
717 ("CTL_FLAG_DATA_IN"));
718
719 #if 0
720 CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
721 request->ip_data_len, io->scsiio.kern_total_len);
722 #endif
723
724 if (io->scsiio.kern_sg_entries > 0) {
725 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
726 ctl_sg_count = io->scsiio.kern_sg_entries;
727 } else {
728 ctl_sglist = &ctl_sg_entry;
729 ctl_sglist->addr = io->scsiio.kern_data_ptr;
730 ctl_sglist->len = io->scsiio.kern_data_len;
731 ctl_sg_count = 1;
732 }
733
734 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
735 ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
736 buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
737 else
738 buffer_offset = 0;
739 len = icl_pdu_data_segment_length(request);
740
741 /*
742 * Make sure the offset, as sent by the initiator, matches the offset
743 * we're supposed to be at in the scatter-gather list.
744 */
745 if (buffer_offset >
746 io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
747 buffer_offset + len <=
748 io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
749 CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
750 "expected %zd; dropping connection", buffer_offset,
751 (size_t)io->scsiio.kern_rel_offset +
752 (size_t)io->scsiio.ext_data_filled);
753 ctl_set_data_phase_error(&io->scsiio);
754 cfiscsi_session_terminate(cs);
755 return (true);
756 }
757
758 /*
759 * This is the offset within the PDU data segment, as opposed
760 * to buffer_offset, which is the offset within the task (SCSI
761 * command).
762 */
763 off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
764 buffer_offset;
765
766 /*
767 * Iterate over the scatter/gather segments, filling them with data
768 * from the PDU data segment. Note that this can get called multiple
769 * times for one SCSI command; the cdw structure holds state for the
770 * scatter/gather list.
771 */
772 for (;;) {
773 KASSERT(cdw->cdw_sg_index < ctl_sg_count,
774 ("cdw->cdw_sg_index >= ctl_sg_count"));
775 if (cdw->cdw_sg_len == 0) {
776 cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
777 cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
778 }
779 KASSERT(off <= len, ("len > off"));
780 copy_len = len - off;
781 if (copy_len > cdw->cdw_sg_len)
782 copy_len = cdw->cdw_sg_len;
783
784 icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
785 cdw->cdw_sg_addr += copy_len;
786 cdw->cdw_sg_len -= copy_len;
787 off += copy_len;
788 io->scsiio.ext_data_filled += copy_len;
789 io->scsiio.kern_data_resid -= copy_len;
790
791 if (cdw->cdw_sg_len == 0) {
792 /*
793 * End of current segment.
794 */
795 if (cdw->cdw_sg_index == ctl_sg_count - 1) {
796 /*
797 * Last segment in scatter/gather list.
798 */
799 break;
800 }
801 cdw->cdw_sg_index++;
802 }
803
804 if (off == len) {
805 /*
806 * End of PDU payload.
807 */
808 break;
809 }
810 }
811
812 if (len > off) {
813 /*
814 * In case of unsolicited data, it's possible that the buffer
815 * provided by CTL is smaller than negotiated FirstBurstLength.
816 * Just ignore the superfluous data; will ask for them with R2T
817 * on next call to cfiscsi_datamove().
818 *
819 * This obviously can only happen with SCSI Command PDU.
820 */
821 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
822 ISCSI_BHS_OPCODE_SCSI_COMMAND)
823 return (true);
824
825 CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
826 "expected %zd; dropping connection",
827 icl_pdu_data_segment_length(request), off);
828 ctl_set_data_phase_error(&io->scsiio);
829 cfiscsi_session_terminate(cs);
830 return (true);
831 }
832
833 if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
834 (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
835 CFISCSI_SESSION_WARN(cs, "got the final packet without "
836 "the F flag; flags = 0x%x; dropping connection",
837 bhsdo->bhsdo_flags);
838 ctl_set_data_phase_error(&io->scsiio);
839 cfiscsi_session_terminate(cs);
840 return (true);
841 }
842
843 if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
844 (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
845 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
846 ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
847 CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
848 "transmitted size was %zd bytes instead of %d; "
849 "dropping connection",
850 (size_t)io->scsiio.ext_data_filled,
851 cdw->cdw_r2t_end);
852 ctl_set_data_phase_error(&io->scsiio);
853 cfiscsi_session_terminate(cs);
854 return (true);
855 } else {
856 /*
857 * For SCSI Command PDU, this just means we need to
858 * solicit more data by sending R2T.
859 */
860 return (false);
861 }
862 }
863
864 if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
865 #if 0
866 CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
867 "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
868 #endif
869
870 return (true);
871 }
872
873 return (false);
874 }
875
876 static void
cfiscsi_pdu_handle_data_out(struct icl_pdu * request)877 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
878 {
879 struct iscsi_bhs_data_out *bhsdo;
880 struct cfiscsi_session *cs;
881 struct cfiscsi_data_wait *cdw = NULL;
882 union ctl_io *io;
883 bool done;
884
885 cs = PDU_SESSION(request);
886 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
887
888 CFISCSI_SESSION_LOCK(cs);
889 TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
890 #if 0
891 CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
892 "ttt 0x%x, itt 0x%x",
893 bhsdo->bhsdo_target_transfer_tag,
894 bhsdo->bhsdo_initiator_task_tag,
895 cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
896 #endif
897 if (bhsdo->bhsdo_target_transfer_tag ==
898 cdw->cdw_target_transfer_tag)
899 break;
900 }
901 CFISCSI_SESSION_UNLOCK(cs);
902 if (cdw == NULL) {
903 CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
904 "0x%x, not found; dropping connection",
905 bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
906 icl_pdu_free(request);
907 cfiscsi_session_terminate(cs);
908 return;
909 }
910
911 if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {
912 CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "
913 "DataSN %u, while expected %u; dropping connection",
914 ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);
915 icl_pdu_free(request);
916 cfiscsi_session_terminate(cs);
917 return;
918 }
919 cdw->cdw_datasn += request->ip_additional_pdus + 1;
920
921 io = cdw->cdw_ctl_io;
922 KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
923 ("CTL_FLAG_DATA_IN"));
924
925 done = cfiscsi_handle_data_segment(request, cdw);
926 if (done) {
927 CFISCSI_SESSION_LOCK(cs);
928 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
929 CFISCSI_SESSION_UNLOCK(cs);
930 done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
931 io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
932 cfiscsi_data_wait_free(cs, cdw);
933 io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
934 if (done)
935 ctl_datamove_done(io, false);
936 else
937 cfiscsi_datamove_out(io);
938 }
939
940 icl_pdu_free(request);
941 }
942
943 static void
cfiscsi_pdu_handle_logout_request(struct icl_pdu * request)944 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
945 {
946 struct iscsi_bhs_logout_request *bhslr;
947 struct iscsi_bhs_logout_response *bhslr2;
948 struct icl_pdu *response;
949 struct cfiscsi_session *cs;
950
951 cs = PDU_SESSION(request);
952 bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
953 switch (bhslr->bhslr_reason & 0x7f) {
954 case BHSLR_REASON_CLOSE_SESSION:
955 case BHSLR_REASON_CLOSE_CONNECTION:
956 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
957 if (response == NULL) {
958 CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
959 icl_pdu_free(request);
960 cfiscsi_session_terminate(cs);
961 return;
962 }
963 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
964 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
965 bhslr2->bhslr_flags = 0x80;
966 bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
967 bhslr2->bhslr_initiator_task_tag =
968 bhslr->bhslr_initiator_task_tag;
969 icl_pdu_free(request);
970 cfiscsi_pdu_queue(response);
971 cfiscsi_session_terminate(cs);
972 break;
973 case BHSLR_REASON_REMOVE_FOR_RECOVERY:
974 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
975 if (response == NULL) {
976 CFISCSI_SESSION_WARN(cs,
977 "failed to allocate memory; dropping connection");
978 icl_pdu_free(request);
979 cfiscsi_session_terminate(cs);
980 return;
981 }
982 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
983 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
984 bhslr2->bhslr_flags = 0x80;
985 bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
986 bhslr2->bhslr_initiator_task_tag =
987 bhslr->bhslr_initiator_task_tag;
988 icl_pdu_free(request);
989 cfiscsi_pdu_queue(response);
990 break;
991 default:
992 CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
993 bhslr->bhslr_reason);
994 icl_pdu_free(request);
995 cfiscsi_session_terminate(cs);
996 break;
997 }
998 }
999
1000 static void
cfiscsi_callout(void * context)1001 cfiscsi_callout(void *context)
1002 {
1003 struct icl_pdu *cp;
1004 struct iscsi_bhs_nop_in *bhsni;
1005 struct cfiscsi_session *cs;
1006
1007 cs = context;
1008
1009 if (cs->cs_terminating)
1010 return;
1011
1012 callout_schedule(&cs->cs_callout, 1 * hz);
1013
1014 atomic_add_int(&cs->cs_timeout, 1);
1015
1016 #ifdef ICL_KERNEL_PROXY
1017 if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
1018 if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
1019 CFISCSI_SESSION_WARN(cs, "login timed out after "
1020 "%d seconds; dropping connection", cs->cs_timeout);
1021 cfiscsi_session_terminate(cs);
1022 }
1023 return;
1024 }
1025 #endif
1026
1027 if (ping_timeout <= 0) {
1028 /*
1029 * Pings are disabled. Don't send NOP-In in this case;
1030 * user might have disabled pings to work around problems
1031 * with certain initiators that can't properly handle
1032 * NOP-In, such as iPXE. Reset the timeout, to avoid
1033 * triggering reconnection, should the user decide to
1034 * reenable them.
1035 */
1036 cs->cs_timeout = 0;
1037 return;
1038 }
1039
1040 if (cs->cs_timeout >= ping_timeout) {
1041 CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1042 "dropping connection", ping_timeout);
1043 cfiscsi_session_terminate(cs);
1044 return;
1045 }
1046
1047 /*
1048 * If the ping was reset less than one second ago - which means
1049 * that we've received some PDU during the last second - assume
1050 * the traffic flows correctly and don't bother sending a NOP-Out.
1051 *
1052 * (It's 2 - one for one second, and one for incrementing is_timeout
1053 * earlier in this routine.)
1054 */
1055 if (cs->cs_timeout < 2)
1056 return;
1057
1058 cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1059 if (cp == NULL) {
1060 CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1061 return;
1062 }
1063 bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1064 bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1065 bhsni->bhsni_flags = 0x80;
1066 bhsni->bhsni_initiator_task_tag = 0xffffffff;
1067
1068 cfiscsi_pdu_queue(cp);
1069 }
1070
1071 static struct cfiscsi_data_wait *
cfiscsi_data_wait_new(struct cfiscsi_session * cs,union ctl_io * io,uint32_t initiator_task_tag,uint32_t * target_transfer_tagp)1072 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,
1073 uint32_t initiator_task_tag, uint32_t *target_transfer_tagp)
1074 {
1075 struct cfiscsi_data_wait *cdw;
1076 int error;
1077
1078 cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
1079 if (cdw == NULL) {
1080 CFISCSI_SESSION_WARN(cs,
1081 "failed to allocate %zd bytes", sizeof(*cdw));
1082 return (NULL);
1083 }
1084
1085 error = icl_conn_transfer_setup(cs->cs_conn, PRIV_REQUEST(io), io,
1086 target_transfer_tagp, &cdw->cdw_icl_prv);
1087 if (error != 0) {
1088 CFISCSI_SESSION_WARN(cs,
1089 "icl_conn_transfer_setup() failed with error %d", error);
1090 uma_zfree(cfiscsi_data_wait_zone, cdw);
1091 return (NULL);
1092 }
1093
1094 cdw->cdw_ctl_io = io;
1095 cdw->cdw_target_transfer_tag = *target_transfer_tagp;
1096 cdw->cdw_initiator_task_tag = initiator_task_tag;
1097
1098 return (cdw);
1099 }
1100
1101 static void
cfiscsi_data_wait_free(struct cfiscsi_session * cs,struct cfiscsi_data_wait * cdw)1102 cfiscsi_data_wait_free(struct cfiscsi_session *cs,
1103 struct cfiscsi_data_wait *cdw)
1104 {
1105
1106 icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);
1107 uma_zfree(cfiscsi_data_wait_zone, cdw);
1108 }
1109
1110 static void
cfiscsi_data_wait_abort(struct cfiscsi_session * cs,struct cfiscsi_data_wait * cdw,int status)1111 cfiscsi_data_wait_abort(struct cfiscsi_session *cs,
1112 struct cfiscsi_data_wait *cdw, int status)
1113 {
1114 union ctl_io *cdw_io;
1115
1116 /*
1117 * Set nonzero port status; this prevents backends from
1118 * assuming that the data transfer actually succeeded
1119 * and writing uninitialized data to disk.
1120 */
1121 MPASS(status != 0);
1122 cdw_io = cdw->cdw_ctl_io;
1123 cdw_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1124 cdw_io->scsiio.io_hdr.port_status = status;
1125 cfiscsi_data_wait_free(cs, cdw);
1126 ctl_datamove_done(cdw_io, false);
1127 }
1128
1129 static void
cfiscsi_session_terminate_tasks(struct cfiscsi_session * cs)1130 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1131 {
1132 struct cfiscsi_data_wait *cdw;
1133 struct icl_pdu *ip;
1134 union ctl_io *io;
1135 int error, last, wait;
1136
1137 if (cs->cs_target == NULL)
1138 return; /* No target yet, so nothing to do. */
1139 ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
1140 ip->ip_bhs->bhs_opcode = ISCSI_BHS_OPCODE_INTERNAL;
1141 io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1142 ctl_zero_io(io);
1143 PRIV_REQUEST(io) = ip;
1144 io->io_hdr.io_type = CTL_IO_TASK;
1145 io->io_hdr.nexus.initid = cs->cs_ctl_initid;
1146 io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1147 io->io_hdr.nexus.targ_lun = 0;
1148 io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1149 io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1150 wait = cs->cs_outstanding_ctl_pdus;
1151 refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1152 error = ctl_run(io);
1153 if (error != CTL_RETVAL_COMPLETE) {
1154 CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d", error);
1155 refcount_release(&cs->cs_outstanding_ctl_pdus);
1156 ctl_free_io(io);
1157 icl_pdu_free(ip);
1158 }
1159
1160 CFISCSI_SESSION_LOCK(cs);
1161 cs->cs_terminating_tasks = true;
1162 while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1163 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1164 CFISCSI_SESSION_UNLOCK(cs);
1165 cfiscsi_data_wait_abort(cs, cdw, 42);
1166 CFISCSI_SESSION_LOCK(cs);
1167 }
1168 CFISCSI_SESSION_UNLOCK(cs);
1169
1170 /*
1171 * Wait for CTL to terminate all the tasks.
1172 */
1173 if (wait > 0)
1174 CFISCSI_SESSION_WARN(cs,
1175 "waiting for CTL to terminate %d tasks", wait);
1176 for (;;) {
1177 refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1178 last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1179 if (last != 0)
1180 break;
1181 tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1182 0, "cfiscsi_terminate", hz / 100);
1183 }
1184 if (wait > 0)
1185 CFISCSI_SESSION_WARN(cs, "tasks terminated");
1186 }
1187
1188 static void
cfiscsi_maintenance_thread(void * arg)1189 cfiscsi_maintenance_thread(void *arg)
1190 {
1191 struct cfiscsi_session *cs;
1192
1193 cs = arg;
1194
1195 for (;;) {
1196 CFISCSI_SESSION_LOCK(cs);
1197 if (cs->cs_terminating == false || cs->cs_handoff_in_progress)
1198 cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1199 CFISCSI_SESSION_UNLOCK(cs);
1200
1201 if (cs->cs_terminating && cs->cs_handoff_in_progress == false) {
1202 /*
1203 * We used to wait up to 30 seconds to deliver queued
1204 * PDUs to the initiator. We also tried hard to deliver
1205 * SCSI Responses for the aborted PDUs. We don't do
1206 * that anymore. We might need to revisit that.
1207 */
1208 callout_drain(&cs->cs_callout);
1209 icl_conn_close(cs->cs_conn);
1210
1211 /*
1212 * At this point ICL receive thread is no longer
1213 * running; no new tasks can be queued.
1214 */
1215 cfiscsi_session_terminate_tasks(cs);
1216 cfiscsi_session_delete(cs);
1217 kthread_exit();
1218 return;
1219 }
1220 CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1221 }
1222 }
1223
1224 static void
cfiscsi_session_terminate(struct cfiscsi_session * cs)1225 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1226 {
1227
1228 cs->cs_terminating = true;
1229 cv_signal(&cs->cs_maintenance_cv);
1230 #ifdef ICL_KERNEL_PROXY
1231 cv_signal(&cs->cs_login_cv);
1232 #endif
1233 }
1234
1235 static int
cfiscsi_session_register_initiator(struct cfiscsi_session * cs)1236 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1237 {
1238 struct cfiscsi_target *ct;
1239 char *name;
1240 int i;
1241
1242 KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1243
1244 ct = cs->cs_target;
1245 name = strdup(cs->cs_initiator_id, M_CTL);
1246 i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1247 if (i < 0) {
1248 CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1249 i);
1250 cs->cs_ctl_initid = -1;
1251 return (1);
1252 }
1253 cs->cs_ctl_initid = i;
1254 #if 0
1255 CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1256 #endif
1257
1258 return (0);
1259 }
1260
1261 static void
cfiscsi_session_unregister_initiator(struct cfiscsi_session * cs)1262 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1263 {
1264 int error;
1265
1266 if (cs->cs_ctl_initid == -1)
1267 return;
1268
1269 error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1270 if (error != 0) {
1271 CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1272 error);
1273 }
1274 cs->cs_ctl_initid = -1;
1275 }
1276
1277 static struct cfiscsi_session *
cfiscsi_session_new(struct cfiscsi_softc * softc,const char * offload)1278 cfiscsi_session_new(struct cfiscsi_softc *softc, const char *offload)
1279 {
1280 struct cfiscsi_session *cs;
1281 int error;
1282
1283 cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1284 if (cs == NULL) {
1285 CFISCSI_WARN("malloc failed");
1286 return (NULL);
1287 }
1288 cs->cs_ctl_initid = -1;
1289
1290 refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1291 TAILQ_INIT(&cs->cs_waiting_for_data_out);
1292 mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1293 cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1294 #ifdef ICL_KERNEL_PROXY
1295 cv_init(&cs->cs_login_cv, "cfiscsi_login");
1296 #endif
1297
1298 /*
1299 * The purpose of this is to avoid racing with session shutdown.
1300 * Otherwise we could have the maintenance thread call icl_conn_close()
1301 * before we call icl_conn_handoff().
1302 */
1303 cs->cs_handoff_in_progress = true;
1304
1305 cs->cs_conn = icl_new_conn(offload, false, "cfiscsi", &cs->cs_lock);
1306 if (cs->cs_conn == NULL) {
1307 free(cs, M_CFISCSI);
1308 return (NULL);
1309 }
1310 cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1311 cs->cs_conn->ic_error = cfiscsi_error_callback;
1312 cs->cs_conn->ic_prv0 = cs;
1313
1314 error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1315 if (error != 0) {
1316 CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1317 free(cs, M_CFISCSI);
1318 return (NULL);
1319 }
1320
1321 mtx_lock(&softc->lock);
1322 cs->cs_id = ++softc->last_session_id;
1323 TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1324 mtx_unlock(&softc->lock);
1325
1326 /*
1327 * Start pinging the initiator.
1328 */
1329 callout_init(&cs->cs_callout, 1);
1330 callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1331
1332 return (cs);
1333 }
1334
1335 static void
cfiscsi_session_delete(struct cfiscsi_session * cs)1336 cfiscsi_session_delete(struct cfiscsi_session *cs)
1337 {
1338 struct cfiscsi_softc *softc;
1339
1340 softc = &cfiscsi_softc;
1341
1342 KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1343 ("destroying session with outstanding CTL pdus"));
1344 KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1345 ("destroying session with non-empty queue"));
1346
1347 mtx_lock(&softc->lock);
1348 TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1349 mtx_unlock(&softc->lock);
1350
1351 cfiscsi_session_unregister_initiator(cs);
1352 if (cs->cs_target != NULL)
1353 cfiscsi_target_release(cs->cs_target);
1354 icl_conn_close(cs->cs_conn);
1355 icl_conn_free(cs->cs_conn);
1356 free(cs, M_CFISCSI);
1357 cv_signal(&softc->sessions_cv);
1358 }
1359
1360 static int
cfiscsi_init(void)1361 cfiscsi_init(void)
1362 {
1363 struct cfiscsi_softc *softc;
1364
1365 softc = &cfiscsi_softc;
1366 bzero(softc, sizeof(*softc));
1367 mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1368
1369 cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1370 #ifdef ICL_KERNEL_PROXY
1371 cv_init(&softc->accept_cv, "cfiscsi_accept");
1372 #endif
1373 TAILQ_INIT(&softc->sessions);
1374 TAILQ_INIT(&softc->targets);
1375
1376 cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1377 sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1378 UMA_ALIGN_PTR, 0);
1379
1380 return (0);
1381 }
1382
1383 static int
cfiscsi_shutdown(void)1384 cfiscsi_shutdown(void)
1385 {
1386 struct cfiscsi_softc *softc = &cfiscsi_softc;
1387
1388 if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))
1389 return (EBUSY);
1390
1391 uma_zdestroy(cfiscsi_data_wait_zone);
1392 #ifdef ICL_KERNEL_PROXY
1393 cv_destroy(&softc->accept_cv);
1394 #endif
1395 cv_destroy(&softc->sessions_cv);
1396 mtx_destroy(&softc->lock);
1397 return (0);
1398 }
1399
1400 #ifdef ICL_KERNEL_PROXY
1401 static void
cfiscsi_accept(struct socket * so,struct sockaddr * sa,int portal_id)1402 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1403 {
1404 struct cfiscsi_session *cs;
1405
1406 cs = cfiscsi_session_new(&cfiscsi_softc, NULL);
1407 if (cs == NULL) {
1408 CFISCSI_WARN("failed to create session");
1409 return;
1410 }
1411
1412 icl_conn_handoff_sock(cs->cs_conn, so);
1413 cs->cs_initiator_sa = sa;
1414 cs->cs_portal_id = portal_id;
1415 cs->cs_handoff_in_progress = false;
1416 cs->cs_waiting_for_ctld = true;
1417 cv_signal(&cfiscsi_softc.accept_cv);
1418
1419 CFISCSI_SESSION_LOCK(cs);
1420 /*
1421 * Wake up the maintenance thread if we got scheduled for termination
1422 * somewhere between cfiscsi_session_new() and icl_conn_handoff_sock().
1423 */
1424 if (cs->cs_terminating)
1425 cfiscsi_session_terminate(cs);
1426 CFISCSI_SESSION_UNLOCK(cs);
1427 }
1428 #endif
1429
1430 static void
cfiscsi_online(void * arg)1431 cfiscsi_online(void *arg)
1432 {
1433 struct cfiscsi_softc *softc;
1434 struct cfiscsi_target *ct;
1435 int online;
1436
1437 ct = (struct cfiscsi_target *)arg;
1438 softc = ct->ct_softc;
1439
1440 mtx_lock(&softc->lock);
1441 if (ct->ct_online) {
1442 mtx_unlock(&softc->lock);
1443 return;
1444 }
1445 ct->ct_online = 1;
1446 online = softc->online++;
1447 mtx_unlock(&softc->lock);
1448 if (online > 0)
1449 return;
1450
1451 #ifdef ICL_KERNEL_PROXY
1452 if (softc->listener != NULL)
1453 icl_listen_free(softc->listener);
1454 softc->listener = icl_listen_new(cfiscsi_accept);
1455 #endif
1456 }
1457
1458 static void
cfiscsi_offline(void * arg)1459 cfiscsi_offline(void *arg)
1460 {
1461 struct cfiscsi_softc *softc;
1462 struct cfiscsi_target *ct;
1463 struct cfiscsi_session *cs;
1464 int error, online;
1465
1466 ct = (struct cfiscsi_target *)arg;
1467 softc = ct->ct_softc;
1468
1469 mtx_lock(&softc->lock);
1470 if (!ct->ct_online) {
1471 mtx_unlock(&softc->lock);
1472 return;
1473 }
1474 ct->ct_online = 0;
1475 online = --softc->online;
1476
1477 do {
1478 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1479 if (cs->cs_target == ct)
1480 cfiscsi_session_terminate(cs);
1481 }
1482 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1483 if (cs->cs_target == ct)
1484 break;
1485 }
1486 if (cs != NULL) {
1487 error = cv_wait_sig(&softc->sessions_cv, &softc->lock);
1488 if (error != 0) {
1489 CFISCSI_SESSION_DEBUG(cs,
1490 "cv_wait failed with error %d\n", error);
1491 break;
1492 }
1493 }
1494 } while (cs != NULL && ct->ct_online == 0);
1495 mtx_unlock(&softc->lock);
1496 if (online > 0)
1497 return;
1498
1499 #ifdef ICL_KERNEL_PROXY
1500 icl_listen_free(softc->listener);
1501 softc->listener = NULL;
1502 #endif
1503 }
1504
1505 static int
cfiscsi_info(void * arg,struct sbuf * sb)1506 cfiscsi_info(void *arg, struct sbuf *sb)
1507 {
1508 struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1509 int retval;
1510
1511 retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1512 ct->ct_state);
1513 return (retval);
1514 }
1515
1516 static void
cfiscsi_ioctl_handoff(struct ctl_iscsi * ci)1517 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1518 {
1519 struct cfiscsi_softc *softc;
1520 struct cfiscsi_session *cs, *cs2;
1521 struct cfiscsi_target *ct;
1522 struct ctl_iscsi_handoff_params *cihp;
1523 int error;
1524
1525 cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1526 softc = &cfiscsi_softc;
1527
1528 CFISCSI_DEBUG("new connection from %s (%s) to %s",
1529 cihp->initiator_name, cihp->initiator_addr,
1530 cihp->target_name);
1531
1532 ct = cfiscsi_target_find(softc, cihp->target_name,
1533 cihp->portal_group_tag);
1534 if (ct == NULL) {
1535 ci->status = CTL_ISCSI_ERROR;
1536 snprintf(ci->error_str, sizeof(ci->error_str),
1537 "%s: target not found", __func__);
1538 return;
1539 }
1540
1541 #ifdef ICL_KERNEL_PROXY
1542 if (cihp->socket > 0 && cihp->connection_id > 0) {
1543 snprintf(ci->error_str, sizeof(ci->error_str),
1544 "both socket and connection_id set");
1545 ci->status = CTL_ISCSI_ERROR;
1546 cfiscsi_target_release(ct);
1547 return;
1548 }
1549 if (cihp->socket == 0) {
1550 mtx_lock(&cfiscsi_softc.lock);
1551 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1552 if (cs->cs_id == cihp->connection_id)
1553 break;
1554 }
1555 if (cs == NULL) {
1556 mtx_unlock(&cfiscsi_softc.lock);
1557 snprintf(ci->error_str, sizeof(ci->error_str),
1558 "connection not found");
1559 ci->status = CTL_ISCSI_ERROR;
1560 cfiscsi_target_release(ct);
1561 return;
1562 }
1563 mtx_unlock(&cfiscsi_softc.lock);
1564 } else {
1565 #endif
1566 cs = cfiscsi_session_new(softc, cihp->offload);
1567 if (cs == NULL) {
1568 ci->status = CTL_ISCSI_ERROR;
1569 snprintf(ci->error_str, sizeof(ci->error_str),
1570 "%s: cfiscsi_session_new failed", __func__);
1571 cfiscsi_target_release(ct);
1572 return;
1573 }
1574 #ifdef ICL_KERNEL_PROXY
1575 }
1576 #endif
1577
1578 /*
1579 * First PDU of Full Feature phase has the same CmdSN as the last
1580 * PDU from the Login Phase received from the initiator. Thus,
1581 * the -1 below.
1582 */
1583 cs->cs_cmdsn = cihp->cmdsn;
1584 cs->cs_statsn = cihp->statsn;
1585 cs->cs_conn->ic_max_recv_data_segment_length =
1586 cihp->max_recv_data_segment_length;
1587 cs->cs_conn->ic_max_send_data_segment_length =
1588 cihp->max_send_data_segment_length;
1589 cs->cs_max_burst_length = cihp->max_burst_length;
1590 cs->cs_first_burst_length = cihp->first_burst_length;
1591 cs->cs_immediate_data = !!cihp->immediate_data;
1592 if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1593 cs->cs_conn->ic_header_crc32c = true;
1594 if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1595 cs->cs_conn->ic_data_crc32c = true;
1596
1597 strlcpy(cs->cs_initiator_name,
1598 cihp->initiator_name, sizeof(cs->cs_initiator_name));
1599 strlcpy(cs->cs_initiator_addr,
1600 cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1601 strlcpy(cs->cs_initiator_alias,
1602 cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1603 memcpy(cs->cs_initiator_isid,
1604 cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1605 snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1606 "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1607 cihp->initiator_isid[0], cihp->initiator_isid[1],
1608 cihp->initiator_isid[2], cihp->initiator_isid[3],
1609 cihp->initiator_isid[4], cihp->initiator_isid[5]);
1610
1611 mtx_lock(&softc->lock);
1612 if (ct->ct_online == 0) {
1613 mtx_unlock(&softc->lock);
1614 CFISCSI_SESSION_LOCK(cs);
1615 cs->cs_handoff_in_progress = false;
1616 cfiscsi_session_terminate(cs);
1617 CFISCSI_SESSION_UNLOCK(cs);
1618 cfiscsi_target_release(ct);
1619 ci->status = CTL_ISCSI_ERROR;
1620 snprintf(ci->error_str, sizeof(ci->error_str),
1621 "%s: port offline", __func__);
1622 return;
1623 }
1624 cs->cs_target = ct;
1625 mtx_unlock(&softc->lock);
1626
1627 restart:
1628 if (!cs->cs_terminating) {
1629 mtx_lock(&softc->lock);
1630 TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1631 if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1632 cs->cs_target == cs2->cs_target &&
1633 strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1634 if (strcmp(cs->cs_initiator_addr,
1635 cs2->cs_initiator_addr) != 0) {
1636 CFISCSI_SESSION_WARN(cs2,
1637 "session reinstatement from "
1638 "different address %s",
1639 cs->cs_initiator_addr);
1640 } else {
1641 CFISCSI_SESSION_DEBUG(cs2,
1642 "session reinstatement");
1643 }
1644 cfiscsi_session_terminate(cs2);
1645 mtx_unlock(&softc->lock);
1646 pause("cfiscsi_reinstate", 1);
1647 goto restart;
1648 }
1649 }
1650 mtx_unlock(&softc->lock);
1651 }
1652
1653 /*
1654 * Register initiator with CTL.
1655 */
1656 cfiscsi_session_register_initiator(cs);
1657
1658 #ifdef ICL_KERNEL_PROXY
1659 if (cihp->socket > 0) {
1660 #endif
1661 error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1662 if (error != 0) {
1663 CFISCSI_SESSION_LOCK(cs);
1664 cs->cs_handoff_in_progress = false;
1665 cfiscsi_session_terminate(cs);
1666 CFISCSI_SESSION_UNLOCK(cs);
1667 ci->status = CTL_ISCSI_ERROR;
1668 snprintf(ci->error_str, sizeof(ci->error_str),
1669 "%s: icl_conn_handoff failed with error %d",
1670 __func__, error);
1671 return;
1672 }
1673 #ifdef ICL_KERNEL_PROXY
1674 }
1675 #endif
1676
1677 #ifdef ICL_KERNEL_PROXY
1678 cs->cs_login_phase = false;
1679
1680 /*
1681 * First PDU of the Full Feature phase has likely already arrived.
1682 * We have to pick it up and execute properly.
1683 */
1684 if (cs->cs_login_pdu != NULL) {
1685 CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1686 cfiscsi_pdu_handle(cs->cs_login_pdu);
1687 cs->cs_login_pdu = NULL;
1688 }
1689 #endif
1690
1691 CFISCSI_SESSION_LOCK(cs);
1692 cs->cs_handoff_in_progress = false;
1693
1694 /*
1695 * Wake up the maintenance thread if we got scheduled for termination.
1696 */
1697 if (cs->cs_terminating)
1698 cfiscsi_session_terminate(cs);
1699 CFISCSI_SESSION_UNLOCK(cs);
1700
1701 ci->status = CTL_ISCSI_OK;
1702 }
1703
1704 static void
cfiscsi_ioctl_list(struct ctl_iscsi * ci)1705 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1706 {
1707 struct ctl_iscsi_list_params *cilp;
1708 struct cfiscsi_session *cs;
1709 struct cfiscsi_softc *softc;
1710 struct sbuf *sb;
1711 int error;
1712
1713 cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1714 softc = &cfiscsi_softc;
1715
1716 sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1717 if (sb == NULL) {
1718 ci->status = CTL_ISCSI_ERROR;
1719 snprintf(ci->error_str, sizeof(ci->error_str),
1720 "Unable to allocate %d bytes for iSCSI session list",
1721 cilp->alloc_len);
1722 return;
1723 }
1724
1725 sbuf_printf(sb, "<ctlislist>\n");
1726 mtx_lock(&softc->lock);
1727 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1728 if (cs->cs_target == NULL)
1729 continue;
1730 error = sbuf_printf(sb, "<connection id=\"%d\">"
1731 "<initiator>%s</initiator>"
1732 "<initiator_addr>%s</initiator_addr>"
1733 "<initiator_alias>%s</initiator_alias>"
1734 "<target>%s</target>"
1735 "<target_alias>%s</target_alias>"
1736 "<target_portal_group_tag>%u</target_portal_group_tag>"
1737 "<header_digest>%s</header_digest>"
1738 "<data_digest>%s</data_digest>"
1739 "<max_recv_data_segment_length>%d</max_recv_data_segment_length>"
1740 "<max_send_data_segment_length>%d</max_send_data_segment_length>"
1741 "<max_burst_length>%d</max_burst_length>"
1742 "<first_burst_length>%d</first_burst_length>"
1743 "<immediate_data>%d</immediate_data>"
1744 "<iser>%d</iser>"
1745 "<offload>%s</offload>"
1746 "</connection>\n",
1747 cs->cs_id,
1748 cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1749 cs->cs_target->ct_name, cs->cs_target->ct_alias,
1750 cs->cs_target->ct_tag,
1751 cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1752 cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1753 cs->cs_conn->ic_max_recv_data_segment_length,
1754 cs->cs_conn->ic_max_send_data_segment_length,
1755 cs->cs_max_burst_length,
1756 cs->cs_first_burst_length,
1757 cs->cs_immediate_data,
1758 cs->cs_conn->ic_iser,
1759 cs->cs_conn->ic_offload);
1760 if (error != 0)
1761 break;
1762 }
1763 mtx_unlock(&softc->lock);
1764 error = sbuf_printf(sb, "</ctlislist>\n");
1765 if (error != 0) {
1766 sbuf_delete(sb);
1767 ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1768 snprintf(ci->error_str, sizeof(ci->error_str),
1769 "Out of space, %d bytes is too small", cilp->alloc_len);
1770 return;
1771 }
1772 sbuf_finish(sb);
1773
1774 error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1775 if (error != 0) {
1776 sbuf_delete(sb);
1777 snprintf(ci->error_str, sizeof(ci->error_str),
1778 "copyout failed with error %d", error);
1779 ci->status = CTL_ISCSI_ERROR;
1780 return;
1781 }
1782 cilp->fill_len = sbuf_len(sb) + 1;
1783 ci->status = CTL_ISCSI_OK;
1784 sbuf_delete(sb);
1785 }
1786
1787 static void
cfiscsi_ioctl_logout(struct ctl_iscsi * ci)1788 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1789 {
1790 struct icl_pdu *response;
1791 struct iscsi_bhs_asynchronous_message *bhsam;
1792 struct ctl_iscsi_logout_params *cilp;
1793 struct cfiscsi_session *cs;
1794 struct cfiscsi_softc *softc;
1795 int found = 0;
1796
1797 cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1798 softc = &cfiscsi_softc;
1799
1800 mtx_lock(&softc->lock);
1801 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1802 if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1803 strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1804 strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1805 continue;
1806
1807 response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1808 if (response == NULL) {
1809 ci->status = CTL_ISCSI_ERROR;
1810 snprintf(ci->error_str, sizeof(ci->error_str),
1811 "Unable to allocate memory");
1812 mtx_unlock(&softc->lock);
1813 return;
1814 }
1815 bhsam =
1816 (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1817 bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1818 bhsam->bhsam_flags = 0x80;
1819 bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1820 bhsam->bhsam_parameter3 = htons(10);
1821 cfiscsi_pdu_queue(response);
1822 found++;
1823 }
1824 mtx_unlock(&softc->lock);
1825
1826 if (found == 0) {
1827 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1828 snprintf(ci->error_str, sizeof(ci->error_str),
1829 "No matching connections found");
1830 return;
1831 }
1832
1833 ci->status = CTL_ISCSI_OK;
1834 }
1835
1836 static void
cfiscsi_ioctl_terminate(struct ctl_iscsi * ci)1837 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1838 {
1839 struct icl_pdu *response;
1840 struct iscsi_bhs_asynchronous_message *bhsam;
1841 struct ctl_iscsi_terminate_params *citp;
1842 struct cfiscsi_session *cs;
1843 struct cfiscsi_softc *softc;
1844 int found = 0;
1845
1846 citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1847 softc = &cfiscsi_softc;
1848
1849 mtx_lock(&softc->lock);
1850 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1851 if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1852 strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1853 strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1854 continue;
1855
1856 response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1857 if (response == NULL) {
1858 /*
1859 * Oh well. Just terminate the connection.
1860 */
1861 } else {
1862 bhsam = (struct iscsi_bhs_asynchronous_message *)
1863 response->ip_bhs;
1864 bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1865 bhsam->bhsam_flags = 0x80;
1866 bhsam->bhsam_0xffffffff = 0xffffffff;
1867 bhsam->bhsam_async_event =
1868 BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1869 cfiscsi_pdu_queue(response);
1870 }
1871 cfiscsi_session_terminate(cs);
1872 found++;
1873 }
1874 mtx_unlock(&softc->lock);
1875
1876 if (found == 0) {
1877 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1878 snprintf(ci->error_str, sizeof(ci->error_str),
1879 "No matching connections found");
1880 return;
1881 }
1882
1883 ci->status = CTL_ISCSI_OK;
1884 }
1885
1886 static void
cfiscsi_ioctl_limits(struct ctl_iscsi * ci)1887 cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
1888 {
1889 struct ctl_iscsi_limits_params *cilp;
1890 struct icl_drv_limits idl;
1891 int error;
1892
1893 cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
1894
1895 error = icl_limits(cilp->offload, false, cilp->socket, &idl);
1896 if (error != 0) {
1897 ci->status = CTL_ISCSI_ERROR;
1898 snprintf(ci->error_str, sizeof(ci->error_str),
1899 "%s: icl_limits failed with error %d",
1900 __func__, error);
1901 return;
1902 }
1903
1904 cilp->max_recv_data_segment_length =
1905 idl.idl_max_recv_data_segment_length;
1906 cilp->max_send_data_segment_length =
1907 idl.idl_max_send_data_segment_length;
1908 cilp->max_burst_length = idl.idl_max_burst_length;
1909 cilp->first_burst_length = idl.idl_first_burst_length;
1910
1911 ci->status = CTL_ISCSI_OK;
1912 }
1913
1914 #ifdef ICL_KERNEL_PROXY
1915 static void
cfiscsi_ioctl_listen(struct ctl_iscsi * ci)1916 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1917 {
1918 struct ctl_iscsi_listen_params *cilp;
1919 struct sockaddr *sa;
1920 int error;
1921
1922 cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1923
1924 if (cfiscsi_softc.listener == NULL) {
1925 CFISCSI_DEBUG("no listener");
1926 snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1927 ci->status = CTL_ISCSI_ERROR;
1928 return;
1929 }
1930
1931 error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1932 if (error != 0) {
1933 CFISCSI_DEBUG("getsockaddr, error %d", error);
1934 snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1935 ci->status = CTL_ISCSI_ERROR;
1936 return;
1937 }
1938
1939 error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1940 cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1941 if (error != 0) {
1942 free(sa, M_SONAME);
1943 CFISCSI_DEBUG("icl_listen_add, error %d", error);
1944 snprintf(ci->error_str, sizeof(ci->error_str),
1945 "icl_listen_add failed, error %d", error);
1946 ci->status = CTL_ISCSI_ERROR;
1947 return;
1948 }
1949
1950 ci->status = CTL_ISCSI_OK;
1951 }
1952
1953 static void
cfiscsi_ioctl_accept(struct ctl_iscsi * ci)1954 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1955 {
1956 struct ctl_iscsi_accept_params *ciap;
1957 struct cfiscsi_session *cs;
1958 int error;
1959
1960 ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1961
1962 mtx_lock(&cfiscsi_softc.lock);
1963 for (;;) {
1964 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1965 if (cs->cs_waiting_for_ctld)
1966 break;
1967 }
1968 if (cs != NULL)
1969 break;
1970 error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1971 if (error != 0) {
1972 mtx_unlock(&cfiscsi_softc.lock);
1973 snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1974 ci->status = CTL_ISCSI_ERROR;
1975 return;
1976 }
1977 }
1978 mtx_unlock(&cfiscsi_softc.lock);
1979
1980 cs->cs_waiting_for_ctld = false;
1981 cs->cs_login_phase = true;
1982
1983 ciap->connection_id = cs->cs_id;
1984 ciap->portal_id = cs->cs_portal_id;
1985 ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1986 error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1987 cs->cs_initiator_sa->sa_len);
1988 if (error != 0) {
1989 snprintf(ci->error_str, sizeof(ci->error_str),
1990 "copyout failed with error %d", error);
1991 ci->status = CTL_ISCSI_ERROR;
1992 return;
1993 }
1994
1995 ci->status = CTL_ISCSI_OK;
1996 }
1997
1998 static void
cfiscsi_ioctl_send(struct ctl_iscsi * ci)1999 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
2000 {
2001 struct ctl_iscsi_send_params *cisp;
2002 struct cfiscsi_session *cs;
2003 struct icl_pdu *ip;
2004 size_t datalen;
2005 void *data;
2006 int error;
2007
2008 cisp = (struct ctl_iscsi_send_params *)&(ci->data);
2009
2010 mtx_lock(&cfiscsi_softc.lock);
2011 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2012 if (cs->cs_id == cisp->connection_id)
2013 break;
2014 }
2015 if (cs == NULL) {
2016 mtx_unlock(&cfiscsi_softc.lock);
2017 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
2018 ci->status = CTL_ISCSI_ERROR;
2019 return;
2020 }
2021 mtx_unlock(&cfiscsi_softc.lock);
2022
2023 #if 0
2024 if (cs->cs_login_phase == false)
2025 return (EBUSY);
2026 #endif
2027
2028 if (cs->cs_terminating) {
2029 snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
2030 ci->status = CTL_ISCSI_ERROR;
2031 return;
2032 }
2033
2034 datalen = cisp->data_segment_len;
2035 /*
2036 * XXX
2037 */
2038 //if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
2039 if (datalen > 65535) {
2040 snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
2041 ci->status = CTL_ISCSI_ERROR;
2042 return;
2043 }
2044 if (datalen > 0) {
2045 data = malloc(datalen, M_CFISCSI, M_WAITOK);
2046 error = copyin(cisp->data_segment, data, datalen);
2047 if (error != 0) {
2048 free(data, M_CFISCSI);
2049 snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
2050 ci->status = CTL_ISCSI_ERROR;
2051 return;
2052 }
2053 }
2054
2055 ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
2056 memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
2057 if (datalen > 0) {
2058 icl_pdu_append_data(ip, data, datalen, M_WAITOK);
2059 free(data, M_CFISCSI);
2060 }
2061 CFISCSI_SESSION_LOCK(cs);
2062 icl_pdu_queue(ip);
2063 CFISCSI_SESSION_UNLOCK(cs);
2064 ci->status = CTL_ISCSI_OK;
2065 }
2066
2067 static void
cfiscsi_ioctl_receive(struct ctl_iscsi * ci)2068 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
2069 {
2070 struct ctl_iscsi_receive_params *cirp;
2071 struct cfiscsi_session *cs;
2072 struct icl_pdu *ip;
2073 void *data;
2074 int error;
2075
2076 cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
2077
2078 mtx_lock(&cfiscsi_softc.lock);
2079 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2080 if (cs->cs_id == cirp->connection_id)
2081 break;
2082 }
2083 if (cs == NULL) {
2084 mtx_unlock(&cfiscsi_softc.lock);
2085 snprintf(ci->error_str, sizeof(ci->error_str),
2086 "connection not found");
2087 ci->status = CTL_ISCSI_ERROR;
2088 return;
2089 }
2090 mtx_unlock(&cfiscsi_softc.lock);
2091
2092 #if 0
2093 if (is->is_login_phase == false)
2094 return (EBUSY);
2095 #endif
2096
2097 CFISCSI_SESSION_LOCK(cs);
2098 while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
2099 error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
2100 if (error != 0) {
2101 CFISCSI_SESSION_UNLOCK(cs);
2102 snprintf(ci->error_str, sizeof(ci->error_str),
2103 "interrupted by signal");
2104 ci->status = CTL_ISCSI_ERROR;
2105 return;
2106 }
2107 }
2108
2109 if (cs->cs_terminating) {
2110 CFISCSI_SESSION_UNLOCK(cs);
2111 snprintf(ci->error_str, sizeof(ci->error_str),
2112 "connection terminating");
2113 ci->status = CTL_ISCSI_ERROR;
2114 return;
2115 }
2116 ip = cs->cs_login_pdu;
2117 cs->cs_login_pdu = NULL;
2118 CFISCSI_SESSION_UNLOCK(cs);
2119
2120 if (ip->ip_data_len > cirp->data_segment_len) {
2121 icl_pdu_free(ip);
2122 snprintf(ci->error_str, sizeof(ci->error_str),
2123 "data segment too big");
2124 ci->status = CTL_ISCSI_ERROR;
2125 return;
2126 }
2127
2128 copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
2129 if (ip->ip_data_len > 0) {
2130 data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
2131 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
2132 copyout(data, cirp->data_segment, ip->ip_data_len);
2133 free(data, M_CFISCSI);
2134 }
2135
2136 icl_pdu_free(ip);
2137 ci->status = CTL_ISCSI_OK;
2138 }
2139
2140 #endif /* !ICL_KERNEL_PROXY */
2141
2142 static void
cfiscsi_ioctl_port_create(struct ctl_req * req)2143 cfiscsi_ioctl_port_create(struct ctl_req *req)
2144 {
2145 struct cfiscsi_target *ct;
2146 struct ctl_port *port;
2147 const char *target, *alias, *val;
2148 struct scsi_vpd_id_descriptor *desc;
2149 int retval, len, idlen;
2150 uint16_t tag;
2151
2152 target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2153 if (target == NULL) {
2154 req->status = CTL_LUN_ERROR;
2155 snprintf(req->error_str, sizeof(req->error_str),
2156 "Missing required argument: cfiscsi_target");
2157 return;
2158 }
2159
2160 val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2161 NULL);
2162 if (val == NULL) {
2163 req->status = CTL_LUN_ERROR;
2164 snprintf(req->error_str, sizeof(req->error_str),
2165 "Missing required argument: cfiscsi_portal_group_tag");
2166 return;
2167 }
2168
2169 alias = dnvlist_get_string(req->args_nvl, "cfiscsi_target_alias", NULL);
2170
2171 tag = strtoul(val, NULL, 0);
2172 ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
2173 if (ct == NULL) {
2174 req->status = CTL_LUN_ERROR;
2175 snprintf(req->error_str, sizeof(req->error_str),
2176 "failed to create target \"%s\"", target);
2177 return;
2178 }
2179 if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
2180 req->status = CTL_LUN_ERROR;
2181 snprintf(req->error_str, sizeof(req->error_str),
2182 "target \"%s\" for portal group tag %u already exists",
2183 target, tag);
2184 cfiscsi_target_release(ct);
2185 return;
2186 }
2187 port = &ct->ct_port;
2188 // WAT
2189 if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
2190 goto done;
2191
2192 port->frontend = &cfiscsi_frontend;
2193 port->port_type = CTL_PORT_ISCSI;
2194 /* XXX KDM what should the real number be here? */
2195 port->num_requested_ctl_io = 4096;
2196 port->port_name = "iscsi";
2197 port->physical_port = (int)tag;
2198 port->virtual_port = ct->ct_target_id;
2199 port->port_online = cfiscsi_online;
2200 port->port_offline = cfiscsi_offline;
2201 port->port_info = cfiscsi_info;
2202 port->onoff_arg = ct;
2203 port->fe_datamove = cfiscsi_datamove;
2204 port->fe_done = cfiscsi_done;
2205 port->targ_port = -1;
2206 port->options = nvlist_clone(req->args_nvl);
2207
2208 /* Generate Port ID. */
2209 idlen = strlen(target) + strlen(",t,0x0001") + 1;
2210 idlen = roundup2(idlen, 4);
2211 len = sizeof(struct scsi_vpd_device_id) + idlen;
2212 port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2213 M_CTL, M_WAITOK | M_ZERO);
2214 port->port_devid->len = len;
2215 desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2216 desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2217 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2218 SVPD_ID_TYPE_SCSI_NAME;
2219 desc->length = idlen;
2220 snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);
2221
2222 /* Generate Target ID. */
2223 idlen = strlen(target) + 1;
2224 idlen = roundup2(idlen, 4);
2225 len = sizeof(struct scsi_vpd_device_id) + idlen;
2226 port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2227 M_CTL, M_WAITOK | M_ZERO);
2228 port->target_devid->len = len;
2229 desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2230 desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2231 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2232 SVPD_ID_TYPE_SCSI_NAME;
2233 desc->length = idlen;
2234 strlcpy(desc->identifier, target, idlen);
2235
2236 retval = ctl_port_register(port);
2237 if (retval != 0) {
2238 free(port->port_devid, M_CFISCSI);
2239 free(port->target_devid, M_CFISCSI);
2240 cfiscsi_target_release(ct);
2241 req->status = CTL_LUN_ERROR;
2242 snprintf(req->error_str, sizeof(req->error_str),
2243 "ctl_port_register() failed with error %d", retval);
2244 return;
2245 }
2246 done:
2247 ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2248 req->status = CTL_LUN_OK;
2249 req->result_nvl = nvlist_create(0);
2250 nvlist_add_number(req->result_nvl, "port_id", port->targ_port);
2251 }
2252
2253 static void
cfiscsi_ioctl_port_remove(struct ctl_req * req)2254 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2255 {
2256 struct cfiscsi_target *ct;
2257 const char *target, *val;
2258 uint16_t tag;
2259
2260 target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2261 if (target == NULL) {
2262 req->status = CTL_LUN_ERROR;
2263 snprintf(req->error_str, sizeof(req->error_str),
2264 "Missing required argument: cfiscsi_target");
2265 return;
2266 }
2267
2268 val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2269 NULL);
2270 if (val == NULL) {
2271 req->status = CTL_LUN_ERROR;
2272 snprintf(req->error_str, sizeof(req->error_str),
2273 "Missing required argument: cfiscsi_portal_group_tag");
2274 return;
2275 }
2276
2277 tag = strtoul(val, NULL, 0);
2278 ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
2279 if (ct == NULL) {
2280 req->status = CTL_LUN_ERROR;
2281 snprintf(req->error_str, sizeof(req->error_str),
2282 "can't find target \"%s\"", target);
2283 return;
2284 }
2285
2286 ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2287 ctl_port_offline(&ct->ct_port);
2288 cfiscsi_target_release(ct);
2289 cfiscsi_target_release(ct);
2290 req->status = CTL_LUN_OK;
2291 }
2292
2293 static int
cfiscsi_ioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flag,struct thread * td)2294 cfiscsi_ioctl(struct cdev *dev,
2295 u_long cmd, caddr_t addr, int flag, struct thread *td)
2296 {
2297 struct ctl_iscsi *ci;
2298 struct ctl_req *req;
2299
2300 if (cmd == CTL_PORT_REQ) {
2301 req = (struct ctl_req *)addr;
2302 switch (req->reqtype) {
2303 case CTL_REQ_CREATE:
2304 cfiscsi_ioctl_port_create(req);
2305 break;
2306 case CTL_REQ_REMOVE:
2307 cfiscsi_ioctl_port_remove(req);
2308 break;
2309 default:
2310 req->status = CTL_LUN_ERROR;
2311 snprintf(req->error_str, sizeof(req->error_str),
2312 "Unsupported request type %d", req->reqtype);
2313 }
2314 return (0);
2315 }
2316
2317 if (cmd != CTL_ISCSI)
2318 return (ENOTTY);
2319
2320 ci = (struct ctl_iscsi *)addr;
2321 switch (ci->type) {
2322 case CTL_ISCSI_HANDOFF:
2323 cfiscsi_ioctl_handoff(ci);
2324 break;
2325 case CTL_ISCSI_LIST:
2326 cfiscsi_ioctl_list(ci);
2327 break;
2328 case CTL_ISCSI_LOGOUT:
2329 cfiscsi_ioctl_logout(ci);
2330 break;
2331 case CTL_ISCSI_TERMINATE:
2332 cfiscsi_ioctl_terminate(ci);
2333 break;
2334 case CTL_ISCSI_LIMITS:
2335 cfiscsi_ioctl_limits(ci);
2336 break;
2337 #ifdef ICL_KERNEL_PROXY
2338 case CTL_ISCSI_LISTEN:
2339 cfiscsi_ioctl_listen(ci);
2340 break;
2341 case CTL_ISCSI_ACCEPT:
2342 cfiscsi_ioctl_accept(ci);
2343 break;
2344 case CTL_ISCSI_SEND:
2345 cfiscsi_ioctl_send(ci);
2346 break;
2347 case CTL_ISCSI_RECEIVE:
2348 cfiscsi_ioctl_receive(ci);
2349 break;
2350 #else
2351 case CTL_ISCSI_LISTEN:
2352 case CTL_ISCSI_ACCEPT:
2353 case CTL_ISCSI_SEND:
2354 case CTL_ISCSI_RECEIVE:
2355 ci->status = CTL_ISCSI_ERROR;
2356 snprintf(ci->error_str, sizeof(ci->error_str),
2357 "%s: CTL compiled without ICL_KERNEL_PROXY",
2358 __func__);
2359 break;
2360 #endif /* !ICL_KERNEL_PROXY */
2361 default:
2362 ci->status = CTL_ISCSI_ERROR;
2363 snprintf(ci->error_str, sizeof(ci->error_str),
2364 "%s: invalid iSCSI request type %d", __func__, ci->type);
2365 break;
2366 }
2367
2368 return (0);
2369 }
2370
2371 static void
cfiscsi_target_hold(struct cfiscsi_target * ct)2372 cfiscsi_target_hold(struct cfiscsi_target *ct)
2373 {
2374
2375 refcount_acquire(&ct->ct_refcount);
2376 }
2377
2378 static void
cfiscsi_target_release(struct cfiscsi_target * ct)2379 cfiscsi_target_release(struct cfiscsi_target *ct)
2380 {
2381 struct cfiscsi_softc *softc;
2382
2383 softc = ct->ct_softc;
2384 mtx_lock(&softc->lock);
2385 if (refcount_release(&ct->ct_refcount)) {
2386 TAILQ_REMOVE(&softc->targets, ct, ct_next);
2387 mtx_unlock(&softc->lock);
2388 if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2389 ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2390 if (ctl_port_deregister(&ct->ct_port) != 0)
2391 printf("%s: ctl_port_deregister() failed\n",
2392 __func__);
2393 }
2394 free(ct, M_CFISCSI);
2395
2396 return;
2397 }
2398 mtx_unlock(&softc->lock);
2399 }
2400
2401 static struct cfiscsi_target *
cfiscsi_target_find(struct cfiscsi_softc * softc,const char * name,uint16_t tag)2402 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name, uint16_t tag)
2403 {
2404 struct cfiscsi_target *ct;
2405
2406 mtx_lock(&softc->lock);
2407 TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2408 if (ct->ct_tag != tag ||
2409 strcmp(name, ct->ct_name) != 0 ||
2410 ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2411 continue;
2412 cfiscsi_target_hold(ct);
2413 mtx_unlock(&softc->lock);
2414 return (ct);
2415 }
2416 mtx_unlock(&softc->lock);
2417
2418 return (NULL);
2419 }
2420
2421 static struct cfiscsi_target *
cfiscsi_target_find_or_create(struct cfiscsi_softc * softc,const char * name,const char * alias,uint16_t tag)2422 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2423 const char *alias, uint16_t tag)
2424 {
2425 struct cfiscsi_target *ct, *newct;
2426
2427 if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2428 return (NULL);
2429
2430 newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2431
2432 mtx_lock(&softc->lock);
2433 TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2434 if (ct->ct_tag != tag ||
2435 strcmp(name, ct->ct_name) != 0 ||
2436 ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2437 continue;
2438 cfiscsi_target_hold(ct);
2439 mtx_unlock(&softc->lock);
2440 free(newct, M_CFISCSI);
2441 return (ct);
2442 }
2443
2444 strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2445 if (alias != NULL)
2446 strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2447 newct->ct_tag = tag;
2448 refcount_init(&newct->ct_refcount, 1);
2449 newct->ct_softc = softc;
2450 if (TAILQ_EMPTY(&softc->targets))
2451 softc->last_target_id = 0;
2452 newct->ct_target_id = ++softc->last_target_id;
2453 TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2454 mtx_unlock(&softc->lock);
2455
2456 return (newct);
2457 }
2458
2459 static void
cfiscsi_pdu_done(struct icl_pdu * ip,int error)2460 cfiscsi_pdu_done(struct icl_pdu *ip, int error)
2461 {
2462
2463 if (error != 0)
2464 ; // XXX: Do something on error?
2465 ((ctl_ref)ip->ip_prv0)(ip->ip_prv1, -1);
2466 }
2467
2468 static void
cfiscsi_datamove_in(union ctl_io * io)2469 cfiscsi_datamove_in(union ctl_io *io)
2470 {
2471 struct cfiscsi_session *cs;
2472 struct icl_pdu *request, *response;
2473 const struct iscsi_bhs_scsi_command *bhssc;
2474 struct iscsi_bhs_data_in *bhsdi;
2475 struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2476 size_t len, expected_len, sg_len, buffer_offset;
2477 size_t max_send_data_segment_length;
2478 const char *sg_addr;
2479 icl_pdu_cb cb;
2480 int ctl_sg_count, error, i;
2481
2482 request = PRIV_REQUEST(io);
2483 cs = PDU_SESSION(request);
2484
2485 bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2486 KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2487 ISCSI_BHS_OPCODE_SCSI_COMMAND,
2488 ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2489
2490 if (io->scsiio.kern_sg_entries > 0) {
2491 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2492 ctl_sg_count = io->scsiio.kern_sg_entries;
2493 } else {
2494 ctl_sglist = &ctl_sg_entry;
2495 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2496 ctl_sglist->len = io->scsiio.kern_data_len;
2497 ctl_sg_count = 1;
2498 }
2499
2500 /*
2501 * This is the offset within the current SCSI command; for the first
2502 * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2503 * it will be the sum of lengths of previous ones.
2504 */
2505 buffer_offset = io->scsiio.kern_rel_offset;
2506
2507 /*
2508 * This is the transfer length expected by the initiator. It can be
2509 * different from the amount of data from the SCSI point of view.
2510 */
2511 expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2512
2513 /*
2514 * If the transfer is outside of expected length -- we are done.
2515 */
2516 if (buffer_offset >= expected_len) {
2517 #if 0
2518 CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2519 "already sent the expected len", buffer_offset);
2520 #endif
2521 ctl_datamove_done(io, true);
2522 return;
2523 }
2524
2525 if (io->scsiio.kern_data_ref != NULL)
2526 cb = cfiscsi_pdu_done;
2527 else
2528 cb = NULL;
2529
2530 i = 0;
2531 sg_addr = NULL;
2532 sg_len = 0;
2533 response = NULL;
2534 bhsdi = NULL;
2535 if (cs->cs_conn->ic_hw_isomax != 0)
2536 max_send_data_segment_length = cs->cs_conn->ic_hw_isomax;
2537 else
2538 max_send_data_segment_length =
2539 cs->cs_conn->ic_max_send_data_segment_length;
2540 for (;;) {
2541 if (response == NULL) {
2542 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2543 if (response == NULL) {
2544 CFISCSI_SESSION_WARN(cs, "failed to "
2545 "allocate memory; dropping connection");
2546 ctl_set_busy(&io->scsiio);
2547 ctl_datamove_done(io, true);
2548 cfiscsi_session_terminate(cs);
2549 return;
2550 }
2551 bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2552 bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2553 bhsdi->bhsdi_initiator_task_tag =
2554 bhssc->bhssc_initiator_task_tag;
2555 bhsdi->bhsdi_target_transfer_tag = 0xffffffff;
2556 bhsdi->bhsdi_datasn = htonl(PRIV_EXPDATASN(io));
2557 bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2558 }
2559
2560 KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2561 if (sg_len == 0) {
2562 sg_addr = ctl_sglist[i].addr;
2563 sg_len = ctl_sglist[i].len;
2564 KASSERT(sg_len > 0, ("sg_len <= 0"));
2565 }
2566
2567 len = sg_len;
2568
2569 /*
2570 * Truncate to maximum data segment length.
2571 */
2572 KASSERT(response->ip_data_len < max_send_data_segment_length,
2573 ("ip_data_len %zd >= max_send_data_segment_length %zd",
2574 response->ip_data_len, max_send_data_segment_length));
2575 if (response->ip_data_len + len > max_send_data_segment_length) {
2576 len = max_send_data_segment_length - response->ip_data_len;
2577 KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2578 len, sg_len));
2579 }
2580
2581 /*
2582 * Truncate to expected data transfer length.
2583 */
2584 KASSERT(buffer_offset + response->ip_data_len < expected_len,
2585 ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2586 buffer_offset, response->ip_data_len, expected_len));
2587 if (buffer_offset + response->ip_data_len + len > expected_len) {
2588 CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2589 "to expected data transfer length %zd",
2590 buffer_offset + response->ip_data_len + len, expected_len);
2591 len = expected_len - (buffer_offset + response->ip_data_len);
2592 KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2593 len, sg_len));
2594 }
2595
2596 error = icl_pdu_append_data(response, sg_addr, len,
2597 M_NOWAIT | (cb ? ICL_NOCOPY : 0));
2598 if (error != 0) {
2599 CFISCSI_SESSION_WARN(cs, "failed to "
2600 "allocate memory; dropping connection");
2601 icl_pdu_free(response);
2602 ctl_set_busy(&io->scsiio);
2603 ctl_datamove_done(io, true);
2604 cfiscsi_session_terminate(cs);
2605 return;
2606 }
2607 sg_addr += len;
2608 sg_len -= len;
2609 io->scsiio.kern_data_resid -= len;
2610
2611 KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2612 ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2613 buffer_offset, response->ip_data_len, expected_len));
2614 if (buffer_offset + response->ip_data_len == expected_len) {
2615 /*
2616 * Already have the amount of data the initiator wanted.
2617 */
2618 break;
2619 }
2620
2621 if (sg_len == 0) {
2622 /*
2623 * End of scatter-gather segment;
2624 * proceed to the next one...
2625 */
2626 if (i == ctl_sg_count - 1) {
2627 /*
2628 * ... unless this was the last one.
2629 */
2630 break;
2631 }
2632 i++;
2633 }
2634
2635 if (response->ip_data_len == max_send_data_segment_length) {
2636 /*
2637 * Can't stuff more data into the current PDU;
2638 * queue it. Note that's not enough to check
2639 * for kern_data_resid == 0 instead; there
2640 * may be several Data-In PDUs for the final
2641 * call to cfiscsi_datamove(), and we want
2642 * to set the F flag only on the last of them.
2643 */
2644 buffer_offset += response->ip_data_len;
2645 if (buffer_offset == io->scsiio.kern_total_len ||
2646 buffer_offset == expected_len) {
2647 buffer_offset -= response->ip_data_len;
2648 break;
2649 }
2650 PRIV_EXPDATASN(io) += howmany(response->ip_data_len,
2651 cs->cs_conn->ic_max_send_data_segment_length);
2652 if (cb != NULL) {
2653 response->ip_prv0 = io->scsiio.kern_data_ref;
2654 response->ip_prv1 = io->scsiio.kern_data_arg;
2655 io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2656 }
2657 cfiscsi_pdu_queue_cb(response, cb);
2658 response = NULL;
2659 bhsdi = NULL;
2660 }
2661 }
2662 if (response != NULL) {
2663 buffer_offset += response->ip_data_len;
2664 if (buffer_offset == io->scsiio.kern_total_len ||
2665 buffer_offset == expected_len) {
2666 bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2667 if (io->io_hdr.status == CTL_SUCCESS) {
2668 bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2669 if (io->scsiio.kern_total_len <
2670 ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2671 bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2672 bhsdi->bhsdi_residual_count =
2673 htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2674 io->scsiio.kern_total_len);
2675 } else if (io->scsiio.kern_total_len >
2676 ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2677 bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2678 bhsdi->bhsdi_residual_count =
2679 htonl(io->scsiio.kern_total_len -
2680 ntohl(bhssc->bhssc_expected_data_transfer_length));
2681 }
2682 bhsdi->bhsdi_status = io->scsiio.scsi_status;
2683 io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2684 }
2685 }
2686 KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2687 PRIV_EXPDATASN(io) += howmany(response->ip_data_len,
2688 cs->cs_conn->ic_max_send_data_segment_length);
2689 if (cb != NULL) {
2690 response->ip_prv0 = io->scsiio.kern_data_ref;
2691 response->ip_prv1 = io->scsiio.kern_data_arg;
2692 io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2693 }
2694 cfiscsi_pdu_queue_cb(response, cb);
2695 }
2696
2697 ctl_datamove_done(io, true);
2698 }
2699
2700 static void
cfiscsi_datamove_out(union ctl_io * io)2701 cfiscsi_datamove_out(union ctl_io *io)
2702 {
2703 struct cfiscsi_session *cs;
2704 struct icl_pdu *request, *response;
2705 const struct iscsi_bhs_scsi_command *bhssc;
2706 struct iscsi_bhs_r2t *bhsr2t;
2707 struct cfiscsi_data_wait *cdw;
2708 struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2709 uint32_t expected_len, datamove_len, r2t_off, r2t_len;
2710 uint32_t target_transfer_tag;
2711 bool done;
2712
2713 request = PRIV_REQUEST(io);
2714 cs = PDU_SESSION(request);
2715
2716 bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2717 KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2718 ISCSI_BHS_OPCODE_SCSI_COMMAND,
2719 ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2720
2721 /*
2722 * Complete write underflow. Not a single byte to read. Return.
2723 */
2724 expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2725 if (io->scsiio.kern_rel_offset >= expected_len) {
2726 ctl_datamove_done(io, true);
2727 return;
2728 }
2729
2730 datamove_len = MIN(io->scsiio.kern_data_len,
2731 expected_len - io->scsiio.kern_rel_offset);
2732
2733 target_transfer_tag =
2734 atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2735 if (target_transfer_tag == 0xffffffff) {
2736 target_transfer_tag =
2737 atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2738 }
2739 cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,
2740 &target_transfer_tag);
2741 if (cdw == NULL) {
2742 CFISCSI_SESSION_WARN(cs, "failed to "
2743 "allocate memory; dropping connection");
2744 ctl_set_busy(&io->scsiio);
2745 ctl_datamove_done(io, true);
2746 cfiscsi_session_terminate(cs);
2747 return;
2748 }
2749 #if 0
2750 CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2751 "task tag 0x%x, target transfer tag 0x%x",
2752 bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2753 #endif
2754
2755 cdw->cdw_ctl_io = io;
2756 cdw->cdw_target_transfer_tag = target_transfer_tag;
2757 cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2758 cdw->cdw_r2t_end = datamove_len;
2759 cdw->cdw_datasn = 0;
2760
2761 /* Set initial data pointer for the CDW respecting ext_data_filled. */
2762 if (io->scsiio.kern_sg_entries > 0) {
2763 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2764 } else {
2765 ctl_sglist = &ctl_sg_entry;
2766 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2767 ctl_sglist->len = datamove_len;
2768 }
2769 cdw->cdw_sg_index = 0;
2770 cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2771 cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2772 r2t_off = io->scsiio.ext_data_filled;
2773 while (r2t_off > 0) {
2774 if (r2t_off >= cdw->cdw_sg_len) {
2775 r2t_off -= cdw->cdw_sg_len;
2776 cdw->cdw_sg_index++;
2777 cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2778 cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2779 continue;
2780 }
2781 cdw->cdw_sg_addr += r2t_off;
2782 cdw->cdw_sg_len -= r2t_off;
2783 r2t_off = 0;
2784 }
2785
2786 if (cs->cs_immediate_data &&
2787 io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2788 icl_pdu_data_segment_length(request)) {
2789 done = cfiscsi_handle_data_segment(request, cdw);
2790 if (done) {
2791 cfiscsi_data_wait_free(cs, cdw);
2792 ctl_datamove_done(io, true);
2793 return;
2794 }
2795 }
2796
2797 r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2798 r2t_len = MIN(datamove_len - io->scsiio.ext_data_filled,
2799 cs->cs_max_burst_length);
2800 cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2801
2802 CFISCSI_SESSION_LOCK(cs);
2803 if (cs->cs_terminating_tasks) {
2804 CFISCSI_SESSION_UNLOCK(cs);
2805 KASSERT((io->io_hdr.flags & CTL_FLAG_ABORT) != 0,
2806 ("%s: I/O request %p on termating session %p not aborted",
2807 __func__, io, cs));
2808 CFISCSI_SESSION_WARN(cs, "aborting data_wait for aborted I/O");
2809 cfiscsi_data_wait_abort(cs, cdw, 44);
2810 return;
2811 }
2812 TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2813 CFISCSI_SESSION_UNLOCK(cs);
2814
2815 /*
2816 * XXX: We should limit the number of outstanding R2T PDUs
2817 * per task to MaxOutstandingR2T.
2818 */
2819 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2820 if (response == NULL) {
2821 CFISCSI_SESSION_WARN(cs, "failed to "
2822 "allocate memory; dropping connection");
2823 ctl_set_busy(&io->scsiio);
2824 ctl_datamove_done(io, true);
2825 cfiscsi_session_terminate(cs);
2826 return;
2827 }
2828 io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
2829 bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2830 bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2831 bhsr2t->bhsr2t_flags = 0x80;
2832 bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2833 bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2834 bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2835 /*
2836 * XXX: Here we assume that cfiscsi_datamove() won't ever
2837 * be running concurrently on several CPUs for a given
2838 * command.
2839 */
2840 bhsr2t->bhsr2t_r2tsn = htonl(PRIV_R2TSN(io)++);
2841 /*
2842 * This is the offset within the current SCSI command;
2843 * i.e. for the first call of datamove(), it will be 0,
2844 * and for subsequent ones it will be the sum of lengths
2845 * of previous ones.
2846 *
2847 * The ext_data_filled is to account for unsolicited
2848 * (immediate) data that might have already arrived.
2849 */
2850 bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2851 /*
2852 * This is the total length (sum of S/G lengths) this call
2853 * to cfiscsi_datamove() is supposed to handle, limited by
2854 * MaxBurstLength.
2855 */
2856 bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2857 cfiscsi_pdu_queue(response);
2858 }
2859
2860 static void
cfiscsi_datamove(union ctl_io * io)2861 cfiscsi_datamove(union ctl_io *io)
2862 {
2863
2864 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2865 cfiscsi_datamove_in(io);
2866 else {
2867 /* We hadn't received anything during this datamove yet. */
2868 io->scsiio.ext_data_filled = 0;
2869 cfiscsi_datamove_out(io);
2870 }
2871 }
2872
2873 static void
cfiscsi_scsi_command_done(union ctl_io * io)2874 cfiscsi_scsi_command_done(union ctl_io *io)
2875 {
2876 struct icl_pdu *request, *response;
2877 struct iscsi_bhs_scsi_command *bhssc;
2878 struct iscsi_bhs_scsi_response *bhssr;
2879 #ifdef DIAGNOSTIC
2880 struct cfiscsi_data_wait *cdw;
2881 struct cfiscsi_session *cs;
2882 #endif
2883 uint16_t sense_length;
2884
2885 request = PRIV_REQUEST(io);
2886 bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2887 KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2888 ISCSI_BHS_OPCODE_SCSI_COMMAND,
2889 ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2890
2891 //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2892 // bhssc->bhssc_initiator_task_tag);
2893
2894 #ifdef DIAGNOSTIC
2895 cs = PDU_SESSION(request);
2896 CFISCSI_SESSION_LOCK(cs);
2897 TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2898 KASSERT(bhssc->bhssc_initiator_task_tag !=
2899 cdw->cdw_initiator_task_tag, ("dangling cdw"));
2900 CFISCSI_SESSION_UNLOCK(cs);
2901 #endif
2902
2903 /*
2904 * Do not return status for aborted commands.
2905 * There are exceptions, but none supported by CTL yet.
2906 */
2907 if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2908 (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2909 (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2910 ctl_free_io(io);
2911 icl_pdu_free(request);
2912 return;
2913 }
2914
2915 response = cfiscsi_pdu_new_response(request, M_WAITOK);
2916 bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2917 bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2918 bhssr->bhssr_flags = 0x80;
2919 /*
2920 * XXX: We don't deal with bidirectional under/overflows;
2921 * does anything actually support those?
2922 */
2923 if (io->scsiio.kern_total_len <
2924 ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2925 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2926 bhssr->bhssr_residual_count =
2927 htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2928 io->scsiio.kern_total_len);
2929 //CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2930 // ntohl(bhssr->bhssr_residual_count));
2931 } else if (io->scsiio.kern_total_len >
2932 ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2933 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2934 bhssr->bhssr_residual_count = htonl(io->scsiio.kern_total_len -
2935 ntohl(bhssc->bhssc_expected_data_transfer_length));
2936 //CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2937 // ntohl(bhssr->bhssr_residual_count));
2938 }
2939 bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2940 bhssr->bhssr_status = io->scsiio.scsi_status;
2941 bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2942 bhssr->bhssr_expdatasn = htonl(PRIV_EXPDATASN(io));
2943
2944 if (io->scsiio.sense_len > 0) {
2945 #if 0
2946 CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2947 io->scsiio.sense_len);
2948 #endif
2949 sense_length = htons(io->scsiio.sense_len);
2950 icl_pdu_append_data(response,
2951 &sense_length, sizeof(sense_length), M_WAITOK);
2952 icl_pdu_append_data(response,
2953 &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2954 }
2955
2956 ctl_free_io(io);
2957 icl_pdu_free(request);
2958 cfiscsi_pdu_queue(response);
2959 }
2960
2961 static void
cfiscsi_task_management_done(union ctl_io * io)2962 cfiscsi_task_management_done(union ctl_io *io)
2963 {
2964 struct icl_pdu *request, *response;
2965 struct iscsi_bhs_task_management_request *bhstmr;
2966 struct iscsi_bhs_task_management_response *bhstmr2;
2967 struct cfiscsi_data_wait *cdw, *tmpcdw;
2968 struct cfiscsi_session *cs, *tcs;
2969 struct cfiscsi_softc *softc;
2970 int cold_reset = 0;
2971
2972 request = PRIV_REQUEST(io);
2973 cs = PDU_SESSION(request);
2974 bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2975 KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2976 ISCSI_BHS_OPCODE_TASK_REQUEST,
2977 ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2978
2979 #if 0
2980 CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2981 bhstmr->bhstmr_initiator_task_tag,
2982 bhstmr->bhstmr_referenced_task_tag);
2983 #endif
2984
2985 if ((bhstmr->bhstmr_function & ~0x80) ==
2986 BHSTMR_FUNCTION_ABORT_TASK) {
2987 /*
2988 * Make sure we no longer wait for Data-Out for this command.
2989 */
2990 CFISCSI_SESSION_LOCK(cs);
2991 TAILQ_FOREACH_SAFE(cdw,
2992 &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2993 if (bhstmr->bhstmr_referenced_task_tag !=
2994 cdw->cdw_initiator_task_tag)
2995 continue;
2996
2997 #if 0
2998 CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2999 "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
3000 #endif
3001 TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
3002 cdw, cdw_next);
3003 cfiscsi_data_wait_abort(cs, cdw, 43);
3004 }
3005 CFISCSI_SESSION_UNLOCK(cs);
3006 }
3007 if ((bhstmr->bhstmr_function & ~0x80) ==
3008 BHSTMR_FUNCTION_TARGET_COLD_RESET &&
3009 io->io_hdr.status == CTL_SUCCESS)
3010 cold_reset = 1;
3011
3012 response = cfiscsi_pdu_new_response(request, M_WAITOK);
3013 bhstmr2 = (struct iscsi_bhs_task_management_response *)
3014 response->ip_bhs;
3015 bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
3016 bhstmr2->bhstmr_flags = 0x80;
3017 switch (io->taskio.task_status) {
3018 case CTL_TASK_FUNCTION_COMPLETE:
3019 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
3020 break;
3021 case CTL_TASK_FUNCTION_SUCCEEDED:
3022 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;
3023 break;
3024 case CTL_TASK_LUN_DOES_NOT_EXIST:
3025 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;
3026 break;
3027 case CTL_TASK_FUNCTION_NOT_SUPPORTED:
3028 default:
3029 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
3030 break;
3031 }
3032 memcpy(bhstmr2->bhstmr_additional_reponse_information,
3033 io->taskio.task_resp, sizeof(io->taskio.task_resp));
3034 bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
3035
3036 ctl_free_io(io);
3037 icl_pdu_free(request);
3038 cfiscsi_pdu_queue(response);
3039
3040 if (cold_reset) {
3041 softc = cs->cs_target->ct_softc;
3042 mtx_lock(&softc->lock);
3043 TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {
3044 if (tcs->cs_target == cs->cs_target)
3045 cfiscsi_session_terminate(tcs);
3046 }
3047 mtx_unlock(&softc->lock);
3048 }
3049 }
3050
3051 static void
cfiscsi_done(union ctl_io * io)3052 cfiscsi_done(union ctl_io *io)
3053 {
3054 struct icl_pdu *request;
3055 struct cfiscsi_session *cs;
3056
3057 KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
3058 ("invalid CTL status %#x", io->io_hdr.status));
3059
3060 request = PRIV_REQUEST(io);
3061 cs = PDU_SESSION(request);
3062
3063 switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
3064 case ISCSI_BHS_OPCODE_SCSI_COMMAND:
3065 cfiscsi_scsi_command_done(io);
3066 break;
3067 case ISCSI_BHS_OPCODE_TASK_REQUEST:
3068 cfiscsi_task_management_done(io);
3069 break;
3070 case ISCSI_BHS_OPCODE_INTERNAL:
3071 /*
3072 * Implicit task termination has just completed; nothing to do.
3073 */
3074 icl_pdu_free(request);
3075 cs->cs_tasks_aborted = true;
3076 refcount_release(&cs->cs_outstanding_ctl_pdus);
3077 wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
3078 ctl_free_io(io);
3079 return;
3080 default:
3081 panic("cfiscsi_done called with wrong opcode 0x%x",
3082 request->ip_bhs->bhs_opcode);
3083 }
3084
3085 refcount_release(&cs->cs_outstanding_ctl_pdus);
3086 }
3087