1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2005-2011 Daniel Braniss <[email protected]>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD$
29 */
30
31 /*
32 | $Id: iscsivar.h 743 2009-08-08 10:54:53Z danny $
33 */
34 #include <sys/lock.h>
35 #include <sys/mutex.h>
36
37 #define ISCSI_MAX_LUNS 128 // don't touch this
38 #if ISCSI_MAX_LUNS > 8
39 /*
40 | for this to work
41 | sysctl kern.cam.cam_srch_hi=1
42 */
43 #endif
44
45 #ifndef ISCSI_INITIATOR_DEBUG
46 #define ISCSI_INITIATOR_DEBUG 1
47 #endif
48
49 #ifdef ISCSI_INITIATOR_DEBUG
50 extern int iscsi_debug;
51 #define debug(level, fmt, args...) do {if(level <= iscsi_debug)\
52 printf("%s: " fmt "\n", __func__ , ##args);} while(0)
53 #define sdebug(level, fmt, args...) do {if(level <= iscsi_debug)\
54 printf("%d] %s: " fmt "\n", sp->sid, __func__ , ##args);} while(0)
55 #define debug_called(level) do {if(level <= iscsi_debug)\
56 printf("%s: called\n", __func__);} while(0)
57 #else
58 #define debug(level, fmt, args...)
59 #define debug_called(level)
60 #define sdebug(level, fmt, args...)
61 #endif /* ISCSI_INITIATOR_DEBUG */
62
63 #define xdebug(fmt, args...) printf(">>> %s: " fmt "\n", __func__ , ##args)
64
65 #define MAX_SESSIONS ISCSI_MAX_TARGETS
66 #define MAX_PDUS (MAX_SESSIONS*256) // XXX: at the moment this is arbitrary
67
68 typedef uint32_t digest_t(const void *, int len, uint32_t ocrc);
69
70 MALLOC_DECLARE(M_ISCSI);
71 MALLOC_DECLARE(M_ISCSIBUF);
72
73 #define ISOK2DIG(dig, pp) ((dig != NULL) && ((pp->ipdu.bhs.opcode & 0x1f) != ISCSI_LOGIN_CMD))
74
75 #ifndef BIT
76 #define BIT(n) (1 <<(n))
77 #endif
78
79 #define ISC_SM_RUN BIT(0)
80 #define ISC_SM_RUNNING BIT(1)
81
82 #define ISC_LINK_UP BIT(2)
83 #define ISC_CON_RUN BIT(3)
84 #define ISC_CON_RUNNING BIT(4)
85 #define ISC_KILL BIT(5)
86 #define ISC_OQNOTEMPTY BIT(6)
87 #define ISC_OWAITING BIT(7)
88 #define ISC_FFPHASE BIT(8)
89
90 #define ISC_CAMDEVS BIT(9)
91 #define ISC_SCANWAIT BIT(10)
92
93 #define ISC_MEMWAIT BIT(11)
94 #define ISC_SIGNALED BIT(12)
95
96 #define ISC_HOLD BIT(15)
97 #define ISC_HOLDED BIT(16)
98
99 #define ISC_SHUTDOWN BIT(31)
100
101 /*
102 | some stats
103 */
104 struct i_stats {
105 int npdu; // number of pdus malloc'ed.
106 int nrecv; // unprocessed received pdus
107 int nsent; // sent pdus
108
109 int nrsp, max_rsp;
110 int nrsv, max_rsv;
111 int ncsnd, max_csnd;
112 int nisnd, max_isnd;
113 int nwsnd, max_wsnd;
114 int nhld, max_hld;
115
116 struct bintime t_sent;
117 struct bintime t_recv;
118 };
119
120 /*
121 | one per 'session'
122 */
123
124 typedef TAILQ_HEAD(, pduq) queue_t;
125
126 typedef struct isc_session {
127 TAILQ_ENTRY(isc_session) sp_link;
128 int flags;
129 struct cdev *dev;
130 struct socket *soc;
131 struct file *fp;
132 struct thread *td;
133
134 struct proc *proc; // the userland process
135 int signal;
136 struct proc *soc_proc;
137 struct proc *stp; // the sm thread
138
139 struct isc_softc *isc;
140
141 digest_t *hdrDigest; // the digest alg. if any
142 digest_t *dataDigest; // the digest alg. if any
143
144 int sid; // Session ID
145 sn_t sn; // sequence number stuff;
146 int cws; // current window size
147
148 int target_nluns; // this and target_lun are
149 // hopefully temporal till I
150 // figure out a better way.
151 int target_lun[ISCSI_MAX_LUNS/(sizeof(int)*8) + 1];
152
153 struct mtx rsp_mtx;
154 struct mtx rsv_mtx;
155 struct mtx snd_mtx;
156 struct mtx hld_mtx;
157 struct mtx io_mtx;
158 queue_t rsp;
159 queue_t rsv;
160 queue_t csnd;
161 queue_t isnd;
162 queue_t wsnd;
163 queue_t hld;
164
165 isc_opt_t opt; // negotiable values
166
167 struct i_stats stats;
168 bhs_t bhs;
169 struct uio uio;
170 struct iovec iov;
171 /*
172 | cam stuff
173 */
174 struct cam_sim *cam_sim;
175 struct cam_path *cam_path;
176 struct mtx cam_mtx;
177 /*
178 | sysctl stuff
179 */
180 struct sysctl_ctx_list clist;
181 struct sysctl_oid *oid;
182 int douio; //XXX: turn on/off uio on read
183 } isc_session_t;
184
185 typedef struct pduq {
186 TAILQ_ENTRY(pduq) pq_link;
187
188 caddr_t buf;
189 u_int len; // the total length of the pdu
190 pdu_t pdu;
191 union ccb *ccb;
192
193 struct uio uio;
194 struct iovec iov[5]; // XXX: careful ...
195 struct mbuf *mp;
196 struct bintime ts;
197 queue_t *pduq;
198 } pduq_t;
199 /*
200 */
201 struct isc_softc {
202 struct mtx isc_mtx;
203 TAILQ_HEAD(,isc_session) isc_sess;
204 int nsess;
205 struct cdev *dev;
206 char isid[6]; // Initiator Session ID (48 bits)
207 struct unrhdr *unit;
208 struct sx unit_sx;
209
210 uma_zone_t pdu_zone; // pool of free pdu's
211 TAILQ_HEAD(,pduq) freepdu;
212
213 #ifdef ISCSI_INITIATOR_DEBUG
214 int npdu_alloc, npdu_max; // for instrumentation
215 #endif
216 #ifdef DO_EVENTHANDLER
217 eventhandler_tag eh;
218 #endif
219 /*
220 | sysctl stuff
221 */
222 struct sysctl_ctx_list clist;
223 struct sysctl_oid *oid;
224 };
225
226 #ifdef ISCSI_INITIATOR_DEBUG
227 extern struct mtx iscsi_dbg_mtx;
228 #endif
229
230 void isc_start_receiver(isc_session_t *sp);
231 void isc_stop_receiver(isc_session_t *sp);
232
233 int isc_sendPDU(isc_session_t *sp, pduq_t *pq);
234 int isc_qout(isc_session_t *sp, pduq_t *pq);
235 int i_prepPDU(isc_session_t *sp, pduq_t *pq);
236
237 int ism_fullfeature(struct cdev *dev, int flag);
238
239 int i_pdu_flush(isc_session_t *sc);
240 int i_setopt(isc_session_t *sp, isc_opt_t *opt);
241 void i_freeopt(isc_opt_t *opt);
242
243 int ic_init(isc_session_t *sp);
244 void ic_destroy(isc_session_t *sp);
245 void ic_lost_target(isc_session_t *sp, int target);
246 int ic_getCamVals(isc_session_t *sp, iscsi_cam_t *cp);
247
248 void ism_recv(isc_session_t *sp, pduq_t *pq);
249 int ism_start(isc_session_t *sp);
250 void ism_restart(isc_session_t *sp);
251 void ism_stop(isc_session_t *sp);
252
253 int scsi_encap(struct cam_sim *sim, union ccb *ccb);
254 int scsi_decap(isc_session_t *sp, pduq_t *opq, pduq_t *pq);
255 void iscsi_r2t(isc_session_t *sp, pduq_t *opq, pduq_t *pq);
256 void iscsi_done(isc_session_t *sp, pduq_t *opq, pduq_t *pq);
257 void iscsi_reject(isc_session_t *sp, pduq_t *opq, pduq_t *pq);
258 void iscsi_async(isc_session_t *sp, pduq_t *pq);
259 void iscsi_cleanup(isc_session_t *sp);
260 int iscsi_requeue(isc_session_t *sp);
261
262 // Serial Number Arithmetic
263 #define _MAXINCR 0x7FFFFFFF // 2 ^ 31 - 1
264 #define SNA_GT(i1, i2) ((i1 != i2) && (\
265 (i1 < i2 && i2 - i1 > _MAXINCR) ||\
266 (i1 > i2 && i1 - i2 < _MAXINCR))?1: 0)
267
268 /*
269 | inlines
270 */
271 static __inline pduq_t *
pdu_alloc(struct isc_softc * isc,int wait)272 pdu_alloc(struct isc_softc *isc, int wait)
273 {
274 pduq_t *pq;
275
276 pq = (pduq_t *)uma_zalloc(isc->pdu_zone, wait /* M_WAITOK or M_NOWAIT*/);
277 if(pq == NULL) {
278 debug(7, "out of mem");
279 return NULL;
280 }
281 #ifdef ISCSI_INITIATOR_DEBUG
282 mtx_lock(&iscsi_dbg_mtx);
283 isc->npdu_alloc++;
284 if(isc->npdu_alloc > isc->npdu_max)
285 isc->npdu_max = isc->npdu_alloc;
286 mtx_unlock(&iscsi_dbg_mtx);
287 #endif
288 memset(pq, 0, sizeof(pduq_t));
289
290 return pq;
291 }
292
293 static __inline void
pdu_free(struct isc_softc * isc,pduq_t * pq)294 pdu_free(struct isc_softc *isc, pduq_t *pq)
295 {
296 if(pq->mp)
297 m_freem(pq->mp);
298 #ifdef NO_USE_MBUF
299 if(pq->buf != NULL)
300 free(pq->buf, M_ISCSIBUF);
301 #endif
302 uma_zfree(isc->pdu_zone, pq);
303 #ifdef ISCSI_INITIATOR_DEBUG
304 mtx_lock(&iscsi_dbg_mtx);
305 isc->npdu_alloc--;
306 mtx_unlock(&iscsi_dbg_mtx);
307 #endif
308 }
309
310 static __inline void
i_nqueue_rsp(isc_session_t * sp,pduq_t * pq)311 i_nqueue_rsp(isc_session_t *sp, pduq_t *pq)
312 {
313 mtx_lock(&sp->rsp_mtx);
314 if(++sp->stats.nrsp > sp->stats.max_rsp)
315 sp->stats.max_rsp = sp->stats.nrsp;
316 TAILQ_INSERT_TAIL(&sp->rsp, pq, pq_link);
317 mtx_unlock(&sp->rsp_mtx);
318 }
319
320 static __inline pduq_t *
i_dqueue_rsp(isc_session_t * sp)321 i_dqueue_rsp(isc_session_t *sp)
322 {
323 pduq_t *pq;
324
325 mtx_lock(&sp->rsp_mtx);
326 if((pq = TAILQ_FIRST(&sp->rsp)) != NULL) {
327 sp->stats.nrsp--;
328 TAILQ_REMOVE(&sp->rsp, pq, pq_link);
329 }
330 mtx_unlock(&sp->rsp_mtx);
331
332 return pq;
333 }
334
335 static __inline void
i_nqueue_rsv(isc_session_t * sp,pduq_t * pq)336 i_nqueue_rsv(isc_session_t *sp, pduq_t *pq)
337 {
338 mtx_lock(&sp->rsv_mtx);
339 if(++sp->stats.nrsv > sp->stats.max_rsv)
340 sp->stats.max_rsv = sp->stats.nrsv;
341 TAILQ_INSERT_TAIL(&sp->rsv, pq, pq_link);
342 mtx_unlock(&sp->rsv_mtx);
343 }
344
345 static __inline pduq_t *
i_dqueue_rsv(isc_session_t * sp)346 i_dqueue_rsv(isc_session_t *sp)
347 {
348 pduq_t *pq;
349
350 mtx_lock(&sp->rsv_mtx);
351 if((pq = TAILQ_FIRST(&sp->rsv)) != NULL) {
352 sp->stats.nrsv--;
353 TAILQ_REMOVE(&sp->rsv, pq, pq_link);
354 }
355 mtx_unlock(&sp->rsv_mtx);
356
357 return pq;
358 }
359
360 static __inline void
i_nqueue_csnd(isc_session_t * sp,pduq_t * pq)361 i_nqueue_csnd(isc_session_t *sp, pduq_t *pq)
362 {
363 mtx_lock(&sp->snd_mtx);
364 if(++sp->stats.ncsnd > sp->stats.max_csnd)
365 sp->stats.max_csnd = sp->stats.ncsnd;
366 TAILQ_INSERT_TAIL(&sp->csnd, pq, pq_link);
367 mtx_unlock(&sp->snd_mtx);
368 }
369
370 static __inline pduq_t *
i_dqueue_csnd(isc_session_t * sp)371 i_dqueue_csnd(isc_session_t *sp)
372 {
373 pduq_t *pq;
374
375 mtx_lock(&sp->snd_mtx);
376 if((pq = TAILQ_FIRST(&sp->csnd)) != NULL) {
377 sp->stats.ncsnd--;
378 TAILQ_REMOVE(&sp->csnd, pq, pq_link);
379 }
380 mtx_unlock(&sp->snd_mtx);
381
382 return pq;
383 }
384
385 static __inline void
i_nqueue_isnd(isc_session_t * sp,pduq_t * pq)386 i_nqueue_isnd(isc_session_t *sp, pduq_t *pq)
387 {
388 mtx_lock(&sp->snd_mtx);
389 if(++sp->stats.nisnd > sp->stats.max_isnd)
390 sp->stats.max_isnd = sp->stats.nisnd;
391 TAILQ_INSERT_TAIL(&sp->isnd, pq, pq_link);
392 mtx_unlock(&sp->snd_mtx);
393 }
394
395 static __inline pduq_t *
i_dqueue_isnd(isc_session_t * sp)396 i_dqueue_isnd(isc_session_t *sp)
397 {
398 pduq_t *pq;
399
400 mtx_lock(&sp->snd_mtx);
401 if((pq = TAILQ_FIRST(&sp->isnd)) != NULL) {
402 sp->stats.nisnd--;
403 TAILQ_REMOVE(&sp->isnd, pq, pq_link);
404 }
405 mtx_unlock(&sp->snd_mtx);
406
407 return pq;
408 }
409
410 static __inline void
i_nqueue_wsnd(isc_session_t * sp,pduq_t * pq)411 i_nqueue_wsnd(isc_session_t *sp, pduq_t *pq)
412 {
413 mtx_lock(&sp->snd_mtx);
414 if(++sp->stats.nwsnd > sp->stats.max_wsnd)
415 sp->stats.max_wsnd = sp->stats.nwsnd;
416 TAILQ_INSERT_TAIL(&sp->wsnd, pq, pq_link);
417 mtx_unlock(&sp->snd_mtx);
418 }
419
420 static __inline pduq_t *
i_dqueue_wsnd(isc_session_t * sp)421 i_dqueue_wsnd(isc_session_t *sp)
422 {
423 pduq_t *pq;
424
425 mtx_lock(&sp->snd_mtx);
426 if((pq = TAILQ_FIRST(&sp->wsnd)) != NULL) {
427 sp->stats.nwsnd--;
428 TAILQ_REMOVE(&sp->wsnd, pq, pq_link);
429 }
430 mtx_unlock(&sp->snd_mtx);
431
432 return pq;
433 }
434
435 static __inline pduq_t *
i_dqueue_snd(isc_session_t * sp,int which)436 i_dqueue_snd(isc_session_t *sp, int which)
437 {
438 pduq_t *pq;
439
440 pq = NULL;
441 mtx_lock(&sp->snd_mtx);
442 if((which & BIT(0)) && (pq = TAILQ_FIRST(&sp->isnd)) != NULL) {
443 sp->stats.nisnd--;
444 TAILQ_REMOVE(&sp->isnd, pq, pq_link);
445 pq->pduq = &sp->isnd; // remember where you came from
446 } else
447 if((which & BIT(1)) && (pq = TAILQ_FIRST(&sp->wsnd)) != NULL) {
448 sp->stats.nwsnd--;
449 TAILQ_REMOVE(&sp->wsnd, pq, pq_link);
450 pq->pduq = &sp->wsnd; // remember where you came from
451 } else
452 if((which & BIT(2)) && (pq = TAILQ_FIRST(&sp->csnd)) != NULL) {
453 sp->stats.ncsnd--;
454 TAILQ_REMOVE(&sp->csnd, pq, pq_link);
455 pq->pduq = &sp->csnd; // remember where you came from
456 }
457 mtx_unlock(&sp->snd_mtx);
458
459 return pq;
460 }
461
462 static __inline void
i_rqueue_pdu(isc_session_t * sp,pduq_t * pq)463 i_rqueue_pdu(isc_session_t *sp, pduq_t *pq)
464 {
465 mtx_lock(&sp->snd_mtx);
466 KASSERT(pq->pduq != NULL, ("pq->pduq is NULL"));
467 TAILQ_INSERT_TAIL(pq->pduq, pq, pq_link);
468 mtx_unlock(&sp->snd_mtx);
469 }
470
471 /*
472 | Waiting for ACK (or something :-)
473 */
474 static __inline void
i_nqueue_hld(isc_session_t * sp,pduq_t * pq)475 i_nqueue_hld(isc_session_t *sp, pduq_t *pq)
476 {
477 getbintime(&pq->ts);
478 mtx_lock(&sp->hld_mtx);
479 if(++sp->stats.nhld > sp->stats.max_hld)
480 sp->stats.max_hld = sp->stats.nhld;
481 TAILQ_INSERT_TAIL(&sp->hld, pq, pq_link);
482 mtx_unlock(&sp->hld_mtx);
483 return;
484 }
485
486 static __inline void
i_remove_hld(isc_session_t * sp,pduq_t * pq)487 i_remove_hld(isc_session_t *sp, pduq_t *pq)
488 {
489 mtx_lock(&sp->hld_mtx);
490 sp->stats.nhld--;
491 TAILQ_REMOVE(&sp->hld, pq, pq_link);
492 mtx_unlock(&sp->hld_mtx);
493 }
494
495 static __inline pduq_t *
i_dqueue_hld(isc_session_t * sp)496 i_dqueue_hld(isc_session_t *sp)
497 {
498 pduq_t *pq;
499
500 mtx_lock(&sp->hld_mtx);
501 if((pq = TAILQ_FIRST(&sp->hld)) != NULL) {
502 sp->stats.nhld--;
503 TAILQ_REMOVE(&sp->hld, pq, pq_link);
504 }
505 mtx_unlock(&sp->hld_mtx);
506
507 return pq;
508 }
509
510 static __inline pduq_t *
i_search_hld(isc_session_t * sp,int itt,int keep)511 i_search_hld(isc_session_t *sp, int itt, int keep)
512 {
513 pduq_t *pq, *tmp;
514
515 pq = NULL;
516
517 mtx_lock(&sp->hld_mtx);
518 TAILQ_FOREACH_SAFE(pq, &sp->hld, pq_link, tmp) {
519 if(pq->pdu.ipdu.bhs.itt == itt) {
520 if(!keep) {
521 sp->stats.nhld--;
522 TAILQ_REMOVE(&sp->hld, pq, pq_link);
523 }
524 break;
525 }
526 }
527 mtx_unlock(&sp->hld_mtx);
528
529 return pq;
530 }
531
532 static __inline void
i_acked_hld(isc_session_t * sp,pdu_t * op)533 i_acked_hld(isc_session_t *sp, pdu_t *op)
534 {
535 pduq_t *pq, *tmp;
536 u_int exp = sp->sn.expCmd;
537
538 pq = NULL;
539 mtx_lock(&sp->hld_mtx);
540 TAILQ_FOREACH_SAFE(pq, &sp->hld, pq_link, tmp) {
541 if((op && op->ipdu.bhs.itt == pq->pdu.ipdu.bhs.itt)
542 || (pq->ccb == NULL
543 && (pq->pdu.ipdu.bhs.opcode != ISCSI_WRITE_DATA)
544 && SNA_GT(exp, ntohl(pq->pdu.ipdu.bhs.ExpStSN)))) {
545 sp->stats.nhld--;
546 TAILQ_REMOVE(&sp->hld, pq, pq_link);
547 pdu_free(sp->isc, pq);
548 }
549 }
550 mtx_unlock(&sp->hld_mtx);
551 }
552
553 static __inline void
i_mbufcopy(struct mbuf * mp,caddr_t dp,int len)554 i_mbufcopy(struct mbuf *mp, caddr_t dp, int len)
555 {
556 struct mbuf *m;
557 caddr_t bp;
558
559 for(m = mp; m != NULL; m = m->m_next) {
560 bp = mtod(m, caddr_t);
561 /*
562 | the pdu is word (4 octed) aligned
563 | so len <= packet
564 */
565 memcpy(dp, bp, MIN(len, m->m_len));
566 dp += m->m_len;
567 len -= m->m_len;
568 if(len <= 0)
569 break;
570 }
571 }
572