1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 *
33 * $FreeBSD$
34 */
35 #ifndef __IW_CXGB4_H__
36 #define __IW_CXGB4_H__
37
38 #include <linux/list.h>
39 #include <linux/spinlock.h>
40 #include <linux/idr.h>
41 #include <linux/completion.h>
42 #include <linux/sched.h>
43 #include <linux/pci.h>
44 #include <linux/dma-mapping.h>
45 #include <linux/wait.h>
46 #include <linux/kref.h>
47 #include <linux/timer.h>
48 #include <linux/io.h>
49 #include <sys/vmem.h>
50
51 #include <asm/byteorder.h>
52
53 #include <netinet/in.h>
54 #include <netinet/toecore.h>
55
56 #include <rdma/ib_verbs.h>
57 #include <rdma/iw_cm.h>
58
59 #include "common/common.h"
60 #include "common/t4_msg.h"
61 #include "common/t4_regs.h"
62 #include "common/t4_tcb.h"
63 #include "t4_l2t.h"
64
65 #define DRV_NAME "iw_cxgbe"
66 #define MOD DRV_NAME ":"
67 #define KTR_IW_CXGBE KTR_SPARE3
68
69 extern int c4iw_debug;
70 extern int use_dsgl;
71 extern int inline_threshold;
72
73 #define PDBG(fmt, args...) \
74 do { \
75 if (c4iw_debug) \
76 printf(MOD fmt, ## args); \
77 } while (0)
78
79 #include "t4.h"
80
cplhdr(struct mbuf * m)81 static inline void *cplhdr(struct mbuf *m)
82 {
83 return mtod(m, void*);
84 }
85
86 #define PBL_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.pbl.start)
87 #define RQT_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.rq.start)
88
89 #define C4IW_ID_TABLE_F_RANDOM 1 /* Pseudo-randomize the id's returned */
90 #define C4IW_ID_TABLE_F_EMPTY 2 /* Table is initially empty */
91 #define C4IW_MAX_PAGE_SIZE 0x8000000
92
93 struct c4iw_id_table {
94 u32 flags;
95 u32 start; /* logical minimal id */
96 u32 last; /* hint for find */
97 u32 max;
98 spinlock_t lock;
99 unsigned long *table;
100 };
101
102 struct c4iw_resource {
103 struct c4iw_id_table tpt_table;
104 struct c4iw_id_table qid_table;
105 struct c4iw_id_table pdid_table;
106 };
107
108 struct c4iw_qid_list {
109 struct list_head entry;
110 u32 qid;
111 };
112
113 struct c4iw_dev_ucontext {
114 struct list_head qpids;
115 struct list_head cqids;
116 struct mutex lock;
117 };
118
119 enum c4iw_rdev_flags {
120 T4_FATAL_ERROR = (1<<0),
121 T4_STATUS_PAGE_DISABLED = (1<<1),
122 };
123
124 struct c4iw_stat {
125 u64 total;
126 u64 cur;
127 u64 max;
128 u64 fail;
129 };
130
131 struct c4iw_stats {
132 struct mutex lock;
133 struct c4iw_stat qid;
134 struct c4iw_stat pd;
135 struct c4iw_stat stag;
136 struct c4iw_stat pbl;
137 struct c4iw_stat rqt;
138 };
139
140 struct c4iw_hw_queue {
141 int t4_eq_status_entries;
142 int t4_max_eq_size;
143 int t4_max_iq_size;
144 int t4_max_rq_size;
145 int t4_max_sq_size;
146 int t4_max_qp_depth;
147 int t4_max_cq_depth;
148 int t4_stat_len;
149 };
150
151 struct c4iw_rdev {
152 struct adapter *adap;
153 struct c4iw_resource resource;
154 unsigned long qpshift;
155 u32 qpmask;
156 unsigned long cqshift;
157 u32 cqmask;
158 struct c4iw_dev_ucontext uctx;
159 vmem_t *rqt_arena;
160 vmem_t *pbl_arena;
161 u32 flags;
162 struct c4iw_stats stats;
163 struct c4iw_hw_queue hw_queue;
164 struct t4_dev_status_page *status_page;
165 unsigned long bar2_pa;
166 void __iomem *bar2_kva;
167 unsigned int bar2_len;
168 struct workqueue_struct *free_workq;
169 };
170
c4iw_fatal_error(struct c4iw_rdev * rdev)171 static inline int c4iw_fatal_error(struct c4iw_rdev *rdev)
172 {
173 return rdev->flags & T4_FATAL_ERROR;
174 }
175
c4iw_num_stags(struct c4iw_rdev * rdev)176 static inline int c4iw_num_stags(struct c4iw_rdev *rdev)
177 {
178 return (int)(rdev->adap->vres.stag.size >> 5);
179 }
180
t4_max_fr_depth(struct c4iw_rdev * rdev,bool use_dsgl)181 static inline int t4_max_fr_depth(struct c4iw_rdev *rdev, bool use_dsgl)
182 {
183 if (rdev->adap->params.ulptx_memwrite_dsgl && use_dsgl)
184 return rdev->adap->params.dev_512sgl_mr ? T4_MAX_FR_FW_DSGL_DEPTH : T4_MAX_FR_DSGL_DEPTH;
185 else
186 return T4_MAX_FR_IMMD_DEPTH;
187 }
188
189 #define C4IW_WR_TO (60*HZ)
190
191 struct c4iw_wr_wait {
192 int ret;
193 struct completion completion;
194 };
195
c4iw_init_wr_wait(struct c4iw_wr_wait * wr_waitp)196 static inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
197 {
198 wr_waitp->ret = 0;
199 init_completion(&wr_waitp->completion);
200 }
201
c4iw_wake_up(struct c4iw_wr_wait * wr_waitp,int ret)202 static inline void c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret)
203 {
204 wr_waitp->ret = ret;
205 complete(&wr_waitp->completion);
206 }
207
208 static inline int
c4iw_wait_for_reply(struct c4iw_rdev * rdev,struct c4iw_wr_wait * wr_waitp,u32 hwtid,u32 qpid,struct socket * so,const char * func)209 c4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp,
210 u32 hwtid, u32 qpid, struct socket *so, const char *func)
211 {
212 struct adapter *sc = rdev->adap;
213 unsigned to = C4IW_WR_TO;
214 int ret;
215 int timedout = 0;
216 struct timeval t1, t2;
217
218 if (c4iw_fatal_error(rdev)) {
219 wr_waitp->ret = -EIO;
220 goto out;
221 }
222
223 getmicrotime(&t1);
224 do {
225 /* If waiting for reply in rdma_init()/rdma_fini() threads, then
226 * check if there are any connection errors.
227 */
228 if (so && so->so_error) {
229 wr_waitp->ret = -ECONNRESET;
230 CTR5(KTR_IW_CXGBE, "%s - Connection ERROR %u for sock %p"
231 "tid %u qpid %u", func,
232 so->so_error, so, hwtid, qpid);
233 break;
234 }
235
236 ret = wait_for_completion_timeout(&wr_waitp->completion, to);
237 if (!ret) {
238 getmicrotime(&t2);
239 timevalsub(&t2, &t1);
240 printf("%s - Device %s not responding after %ld.%06ld "
241 "seconds - tid %u qpid %u\n", func,
242 device_get_nameunit(sc->dev), t2.tv_sec, t2.tv_usec,
243 hwtid, qpid);
244 if (c4iw_fatal_error(rdev)) {
245 wr_waitp->ret = -EIO;
246 break;
247 }
248 to = to << 2;
249 timedout = 1;
250 }
251 } while (!ret);
252
253 out:
254 if (timedout) {
255 getmicrotime(&t2);
256 timevalsub(&t2, &t1);
257 printf("%s - Device %s reply after %ld.%06ld seconds - "
258 "tid %u qpid %u\n", func, device_get_nameunit(sc->dev),
259 t2.tv_sec, t2.tv_usec, hwtid, qpid);
260 }
261 if (wr_waitp->ret)
262 CTR4(KTR_IW_CXGBE, "%p: FW reply %d tid %u qpid %u", sc,
263 wr_waitp->ret, hwtid, qpid);
264 return (wr_waitp->ret);
265 }
266
267 struct c4iw_dev {
268 struct ib_device ibdev;
269 struct pci_dev pdev;
270 struct c4iw_rdev rdev;
271 u32 device_cap_flags;
272 struct idr cqidr;
273 struct idr qpidr;
274 struct idr mmidr;
275 spinlock_t lock;
276 struct dentry *debugfs_root;
277 u32 avail_ird;
278 };
279
to_c4iw_dev(struct ib_device * ibdev)280 static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)
281 {
282 return container_of(ibdev, struct c4iw_dev, ibdev);
283 }
284
rdev_to_c4iw_dev(struct c4iw_rdev * rdev)285 static inline struct c4iw_dev *rdev_to_c4iw_dev(struct c4iw_rdev *rdev)
286 {
287 return container_of(rdev, struct c4iw_dev, rdev);
288 }
289
get_chp(struct c4iw_dev * rhp,u32 cqid)290 static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid)
291 {
292 return idr_find(&rhp->cqidr, cqid);
293 }
294
get_qhp(struct c4iw_dev * rhp,u32 qpid)295 static inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid)
296 {
297 return idr_find(&rhp->qpidr, qpid);
298 }
299
get_mhp(struct c4iw_dev * rhp,u32 mmid)300 static inline struct c4iw_mr *get_mhp(struct c4iw_dev *rhp, u32 mmid)
301 {
302 return idr_find(&rhp->mmidr, mmid);
303 }
304
_insert_handle(struct c4iw_dev * rhp,struct idr * idr,void * handle,u32 id,int lock)305 static inline int _insert_handle(struct c4iw_dev *rhp, struct idr *idr,
306 void *handle, u32 id, int lock)
307 {
308 int ret;
309 int newid;
310
311 do {
312 if (!idr_pre_get(idr, lock ? GFP_KERNEL : GFP_ATOMIC))
313 return -ENOMEM;
314 if (lock)
315 spin_lock_irq(&rhp->lock);
316 ret = idr_get_new_above(idr, handle, id, &newid);
317 BUG_ON(!ret && newid != id);
318 if (lock)
319 spin_unlock_irq(&rhp->lock);
320 } while (ret == -EAGAIN);
321
322 return ret;
323 }
324
insert_handle(struct c4iw_dev * rhp,struct idr * idr,void * handle,u32 id)325 static inline int insert_handle(struct c4iw_dev *rhp, struct idr *idr,
326 void *handle, u32 id)
327 {
328 return _insert_handle(rhp, idr, handle, id, 1);
329 }
330
insert_handle_nolock(struct c4iw_dev * rhp,struct idr * idr,void * handle,u32 id)331 static inline int insert_handle_nolock(struct c4iw_dev *rhp, struct idr *idr,
332 void *handle, u32 id)
333 {
334 return _insert_handle(rhp, idr, handle, id, 0);
335 }
336
_remove_handle(struct c4iw_dev * rhp,struct idr * idr,u32 id,int lock)337 static inline void _remove_handle(struct c4iw_dev *rhp, struct idr *idr,
338 u32 id, int lock)
339 {
340 if (lock)
341 spin_lock_irq(&rhp->lock);
342 idr_remove(idr, id);
343 if (lock)
344 spin_unlock_irq(&rhp->lock);
345 }
346
remove_handle(struct c4iw_dev * rhp,struct idr * idr,u32 id)347 static inline void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id)
348 {
349 _remove_handle(rhp, idr, id, 1);
350 }
351
remove_handle_nolock(struct c4iw_dev * rhp,struct idr * idr,u32 id)352 static inline void remove_handle_nolock(struct c4iw_dev *rhp,
353 struct idr *idr, u32 id)
354 {
355 _remove_handle(rhp, idr, id, 0);
356 }
357
358 extern int c4iw_max_read_depth;
359
cur_max_read_depth(struct c4iw_dev * dev)360 static inline int cur_max_read_depth(struct c4iw_dev *dev)
361 {
362 return min(dev->rdev.adap->params.max_ordird_qp, c4iw_max_read_depth);
363 }
364
365 struct c4iw_pd {
366 struct ib_pd ibpd;
367 u32 pdid;
368 struct c4iw_dev *rhp;
369 };
370
to_c4iw_pd(struct ib_pd * ibpd)371 static inline struct c4iw_pd *to_c4iw_pd(struct ib_pd *ibpd)
372 {
373 return container_of(ibpd, struct c4iw_pd, ibpd);
374 }
375
376 struct tpt_attributes {
377 u64 len;
378 u64 va_fbo;
379 enum fw_ri_mem_perms perms;
380 u32 stag;
381 u32 pdid;
382 u32 qpid;
383 u32 pbl_addr;
384 u32 pbl_size;
385 u32 state:1;
386 u32 type:2;
387 u32 rsvd:1;
388 u32 remote_invaliate_disable:1;
389 u32 zbva:1;
390 u32 mw_bind_enable:1;
391 u32 page_size:5;
392 };
393
394 struct c4iw_mr {
395 struct ib_mr ibmr;
396 struct ib_umem *umem;
397 struct c4iw_dev *rhp;
398 u64 kva;
399 struct tpt_attributes attr;
400 u64 *mpl;
401 dma_addr_t mpl_addr;
402 u32 max_mpl_len;
403 u32 mpl_len;
404 };
405
to_c4iw_mr(struct ib_mr * ibmr)406 static inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr)
407 {
408 return container_of(ibmr, struct c4iw_mr, ibmr);
409 }
410
411 struct c4iw_mw {
412 struct ib_mw ibmw;
413 struct c4iw_dev *rhp;
414 u64 kva;
415 struct tpt_attributes attr;
416 };
417
to_c4iw_mw(struct ib_mw * ibmw)418 static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw)
419 {
420 return container_of(ibmw, struct c4iw_mw, ibmw);
421 }
422
423 struct c4iw_cq {
424 struct ib_cq ibcq;
425 struct c4iw_dev *rhp;
426 struct t4_cq cq;
427 spinlock_t lock;
428 spinlock_t comp_handler_lock;
429 atomic_t refcnt;
430 wait_queue_head_t wait;
431 };
432
to_c4iw_cq(struct ib_cq * ibcq)433 static inline struct c4iw_cq *to_c4iw_cq(struct ib_cq *ibcq)
434 {
435 return container_of(ibcq, struct c4iw_cq, ibcq);
436 }
437
438 struct c4iw_mpa_attributes {
439 u8 initiator;
440 u8 recv_marker_enabled;
441 u8 xmit_marker_enabled;
442 u8 crc_enabled;
443 u8 enhanced_rdma_conn;
444 u8 version;
445 u8 p2p_type;
446 };
447
448 struct c4iw_qp_attributes {
449 u32 scq;
450 u32 rcq;
451 u32 sq_num_entries;
452 u32 rq_num_entries;
453 u32 sq_max_sges;
454 u32 sq_max_sges_rdma_write;
455 u32 rq_max_sges;
456 u32 state;
457 u8 enable_rdma_read;
458 u8 enable_rdma_write;
459 u8 enable_bind;
460 u8 enable_mmid0_fastreg;
461 u32 max_ord;
462 u32 max_ird;
463 u32 pd;
464 u32 next_state;
465 char terminate_buffer[52];
466 u32 terminate_msg_len;
467 u8 is_terminate_local;
468 struct c4iw_mpa_attributes mpa_attr;
469 struct c4iw_ep *llp_stream_handle;
470 u8 layer_etype;
471 u8 ecode;
472 u16 sq_db_inc;
473 u16 rq_db_inc;
474 u8 send_term;
475 };
476
477 struct c4iw_qp {
478 struct ib_qp ibqp;
479 struct c4iw_dev *rhp;
480 struct c4iw_ep *ep;
481 struct c4iw_qp_attributes attr;
482 struct t4_wq wq;
483 spinlock_t lock;
484 struct mutex mutex;
485 struct kref kref;
486 wait_queue_head_t wait;
487 struct timer_list timer;
488 int sq_sig_all;
489 struct work_struct free_work;
490 struct c4iw_ucontext *ucontext;
491 };
492
to_c4iw_qp(struct ib_qp * ibqp)493 static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)
494 {
495 return container_of(ibqp, struct c4iw_qp, ibqp);
496 }
497
498 struct c4iw_ucontext {
499 struct ib_ucontext ibucontext;
500 struct c4iw_dev_ucontext uctx;
501 u32 key;
502 spinlock_t mmap_lock;
503 struct list_head mmaps;
504 struct kref kref;
505 };
506
to_c4iw_ucontext(struct ib_ucontext * c)507 static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
508 {
509 return container_of(c, struct c4iw_ucontext, ibucontext);
510 }
511
512 void _c4iw_free_ucontext(struct kref *kref);
513
c4iw_put_ucontext(struct c4iw_ucontext * ucontext)514 static inline void c4iw_put_ucontext(struct c4iw_ucontext *ucontext)
515 {
516 kref_put(&ucontext->kref, _c4iw_free_ucontext);
517 }
c4iw_get_ucontext(struct c4iw_ucontext * ucontext)518 static inline void c4iw_get_ucontext(struct c4iw_ucontext *ucontext)
519 {
520 kref_get(&ucontext->kref);
521 }
522
523 struct c4iw_mm_entry {
524 struct list_head entry;
525 u64 addr;
526 u32 key;
527 unsigned len;
528 };
529
remove_mmap(struct c4iw_ucontext * ucontext,u32 key,unsigned len)530 static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext,
531 u32 key, unsigned len)
532 {
533 struct list_head *pos, *nxt;
534 struct c4iw_mm_entry *mm;
535
536 spin_lock(&ucontext->mmap_lock);
537 list_for_each_safe(pos, nxt, &ucontext->mmaps) {
538
539 mm = list_entry(pos, struct c4iw_mm_entry, entry);
540 if (mm->key == key && mm->len == len) {
541 list_del_init(&mm->entry);
542 spin_unlock(&ucontext->mmap_lock);
543 CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d",
544 __func__, key, (unsigned long long) mm->addr,
545 mm->len);
546 return mm;
547 }
548 }
549 spin_unlock(&ucontext->mmap_lock);
550 return NULL;
551 }
552
insert_mmap(struct c4iw_ucontext * ucontext,struct c4iw_mm_entry * mm)553 static inline void insert_mmap(struct c4iw_ucontext *ucontext,
554 struct c4iw_mm_entry *mm)
555 {
556 spin_lock(&ucontext->mmap_lock);
557 CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d", __func__, mm->key,
558 (unsigned long long) mm->addr, mm->len);
559 list_add_tail(&mm->entry, &ucontext->mmaps);
560 spin_unlock(&ucontext->mmap_lock);
561 }
562
563 enum c4iw_qp_attr_mask {
564 C4IW_QP_ATTR_NEXT_STATE = 1 << 0,
565 C4IW_QP_ATTR_SQ_DB = 1<<1,
566 C4IW_QP_ATTR_RQ_DB = 1<<2,
567 C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7,
568 C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8,
569 C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9,
570 C4IW_QP_ATTR_MAX_ORD = 1 << 11,
571 C4IW_QP_ATTR_MAX_IRD = 1 << 12,
572 C4IW_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22,
573 C4IW_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23,
574 C4IW_QP_ATTR_MPA_ATTR = 1 << 24,
575 C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25,
576 C4IW_QP_ATTR_VALID_MODIFY = (C4IW_QP_ATTR_ENABLE_RDMA_READ |
577 C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
578 C4IW_QP_ATTR_MAX_ORD |
579 C4IW_QP_ATTR_MAX_IRD |
580 C4IW_QP_ATTR_LLP_STREAM_HANDLE |
581 C4IW_QP_ATTR_STREAM_MSG_BUFFER |
582 C4IW_QP_ATTR_MPA_ATTR |
583 C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE)
584 };
585
586 int c4iw_modify_qp(struct c4iw_dev *rhp,
587 struct c4iw_qp *qhp,
588 enum c4iw_qp_attr_mask mask,
589 struct c4iw_qp_attributes *attrs,
590 int internal);
591
592 enum c4iw_qp_state {
593 C4IW_QP_STATE_IDLE,
594 C4IW_QP_STATE_RTS,
595 C4IW_QP_STATE_ERROR,
596 C4IW_QP_STATE_TERMINATE,
597 C4IW_QP_STATE_CLOSING,
598 C4IW_QP_STATE_TOT
599 };
600
601 /*
602 * IW_CXGBE event bits.
603 * These bits are used for handling all events for a particular 'ep' serially.
604 */
605 #define C4IW_EVENT_SOCKET 0x0001
606 #define C4IW_EVENT_TIMEOUT 0x0002
607 #define C4IW_EVENT_TERM 0x0004
608
c4iw_convert_state(enum ib_qp_state ib_state)609 static inline int c4iw_convert_state(enum ib_qp_state ib_state)
610 {
611 switch (ib_state) {
612 case IB_QPS_RESET:
613 case IB_QPS_INIT:
614 return C4IW_QP_STATE_IDLE;
615 case IB_QPS_RTS:
616 return C4IW_QP_STATE_RTS;
617 case IB_QPS_SQD:
618 return C4IW_QP_STATE_CLOSING;
619 case IB_QPS_SQE:
620 return C4IW_QP_STATE_TERMINATE;
621 case IB_QPS_ERR:
622 return C4IW_QP_STATE_ERROR;
623 default:
624 return -1;
625 }
626 }
627
to_ib_qp_state(int c4iw_qp_state)628 static inline int to_ib_qp_state(int c4iw_qp_state)
629 {
630 switch (c4iw_qp_state) {
631 case C4IW_QP_STATE_IDLE:
632 return IB_QPS_INIT;
633 case C4IW_QP_STATE_RTS:
634 return IB_QPS_RTS;
635 case C4IW_QP_STATE_CLOSING:
636 return IB_QPS_SQD;
637 case C4IW_QP_STATE_TERMINATE:
638 return IB_QPS_SQE;
639 case C4IW_QP_STATE_ERROR:
640 return IB_QPS_ERR;
641 }
642 return IB_QPS_ERR;
643 }
644
645 #define C4IW_DRAIN_OPCODE FW_RI_SGE_EC_CR_RETURN
646
c4iw_ib_to_tpt_access(int a)647 static inline u32 c4iw_ib_to_tpt_access(int a)
648 {
649 return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
650 (a & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0) |
651 (a & IB_ACCESS_LOCAL_WRITE ? FW_RI_MEM_ACCESS_LOCAL_WRITE : 0) |
652 FW_RI_MEM_ACCESS_LOCAL_READ;
653 }
654
c4iw_ib_to_tpt_bind_access(int acc)655 static inline u32 c4iw_ib_to_tpt_bind_access(int acc)
656 {
657 return (acc & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
658 (acc & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0);
659 }
660
661 enum c4iw_mmid_state {
662 C4IW_STAG_STATE_VALID,
663 C4IW_STAG_STATE_INVALID
664 };
665
666 #define C4IW_NODE_DESC "iw_cxgbe Chelsio Communications"
667
668 #define MPA_KEY_REQ "MPA ID Req Frame"
669 #define MPA_KEY_REP "MPA ID Rep Frame"
670
671 #define MPA_MAX_PRIVATE_DATA 256
672 #define MPA_ENHANCED_RDMA_CONN 0x10
673 #define MPA_REJECT 0x20
674 #define MPA_CRC 0x40
675 #define MPA_MARKERS 0x80
676 #define MPA_FLAGS_MASK 0xE0
677
678 #define MPA_V2_PEER2PEER_MODEL 0x8000
679 #define MPA_V2_ZERO_LEN_FPDU_RTR 0x4000
680 #define MPA_V2_RDMA_WRITE_RTR 0x8000
681 #define MPA_V2_RDMA_READ_RTR 0x4000
682 #define MPA_V2_IRD_ORD_MASK 0x3FFF
683
684 #define c4iw_put_ep(ep) { \
685 CTR4(KTR_IW_CXGBE, "put_ep (%s:%u) ep %p, refcnt %d", \
686 __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \
687 WARN_ON(atomic_read(&(ep)->kref.refcount) < 1); \
688 kref_put(&((ep)->kref), _c4iw_free_ep); \
689 }
690
691 #define c4iw_get_ep(ep) { \
692 CTR4(KTR_IW_CXGBE, "get_ep (%s:%u) ep %p, refcnt %d", \
693 __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \
694 kref_get(&((ep)->kref)); \
695 }
696
697 void _c4iw_free_ep(struct kref *kref);
698
699 struct mpa_message {
700 u8 key[16];
701 u8 flags;
702 u8 revision;
703 __be16 private_data_size;
704 u8 private_data[0];
705 };
706
707 struct mpa_v2_conn_params {
708 __be16 ird;
709 __be16 ord;
710 };
711
712 struct terminate_message {
713 u8 layer_etype;
714 u8 ecode;
715 __be16 hdrct_rsvd;
716 u8 len_hdrs[0];
717 };
718
719 #define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)
720
721 enum c4iw_layers_types {
722 LAYER_RDMAP = 0x00,
723 LAYER_DDP = 0x10,
724 LAYER_MPA = 0x20,
725 RDMAP_LOCAL_CATA = 0x00,
726 RDMAP_REMOTE_PROT = 0x01,
727 RDMAP_REMOTE_OP = 0x02,
728 DDP_LOCAL_CATA = 0x00,
729 DDP_TAGGED_ERR = 0x01,
730 DDP_UNTAGGED_ERR = 0x02,
731 DDP_LLP = 0x03
732 };
733
734 enum c4iw_rdma_ecodes {
735 RDMAP_INV_STAG = 0x00,
736 RDMAP_BASE_BOUNDS = 0x01,
737 RDMAP_ACC_VIOL = 0x02,
738 RDMAP_STAG_NOT_ASSOC = 0x03,
739 RDMAP_TO_WRAP = 0x04,
740 RDMAP_INV_VERS = 0x05,
741 RDMAP_INV_OPCODE = 0x06,
742 RDMAP_STREAM_CATA = 0x07,
743 RDMAP_GLOBAL_CATA = 0x08,
744 RDMAP_CANT_INV_STAG = 0x09,
745 RDMAP_UNSPECIFIED = 0xff
746 };
747
748 enum c4iw_ddp_ecodes {
749 DDPT_INV_STAG = 0x00,
750 DDPT_BASE_BOUNDS = 0x01,
751 DDPT_STAG_NOT_ASSOC = 0x02,
752 DDPT_TO_WRAP = 0x03,
753 DDPT_INV_VERS = 0x04,
754 DDPU_INV_QN = 0x01,
755 DDPU_INV_MSN_NOBUF = 0x02,
756 DDPU_INV_MSN_RANGE = 0x03,
757 DDPU_INV_MO = 0x04,
758 DDPU_MSG_TOOBIG = 0x05,
759 DDPU_INV_VERS = 0x06
760 };
761
762 enum c4iw_mpa_ecodes {
763 MPA_CRC_ERR = 0x02,
764 MPA_MARKER_ERR = 0x03,
765 MPA_LOCAL_CATA = 0x05,
766 MPA_INSUFF_IRD = 0x06,
767 MPA_NOMATCH_RTR = 0x07,
768 };
769
770 enum c4iw_ep_state {
771 IDLE = 0,
772 LISTEN,
773 CONNECTING,
774 MPA_REQ_WAIT,
775 MPA_REQ_SENT,
776 MPA_REQ_RCVD,
777 MPA_REP_SENT,
778 FPDU_MODE,
779 ABORTING,
780 CLOSING,
781 MORIBUND,
782 DEAD,
783 };
784
785 enum c4iw_ep_flags {
786 PEER_ABORT_IN_PROGRESS = 0,
787 ABORT_REQ_IN_PROGRESS = 1,
788 RELEASE_RESOURCES = 2,
789 CLOSE_SENT = 3,
790 TIMEOUT = 4,
791 QP_REFERENCED = 5,
792 STOP_MPA_TIMER = 7,
793 };
794
795 enum c4iw_ep_history {
796 ACT_OPEN_REQ = 0,
797 ACT_OFLD_CONN = 1,
798 ACT_OPEN_RPL = 2,
799 ACT_ESTAB = 3,
800 PASS_ACCEPT_REQ = 4,
801 PASS_ESTAB = 5,
802 ABORT_UPCALL = 6,
803 ESTAB_UPCALL = 7,
804 CLOSE_UPCALL = 8,
805 ULP_ACCEPT = 9,
806 ULP_REJECT = 10,
807 TIMEDOUT = 11,
808 PEER_ABORT = 12,
809 PEER_CLOSE = 13,
810 CONNREQ_UPCALL = 14,
811 ABORT_CONN = 15,
812 DISCONN_UPCALL = 16,
813 EP_DISC_CLOSE = 17,
814 EP_DISC_ABORT = 18,
815 CONN_RPL_UPCALL = 19,
816 ACT_RETRY_NOMEM = 20,
817 ACT_RETRY_INUSE = 21,
818 CLOSE_CON_RPL = 22,
819 EP_DISC_FAIL = 24,
820 QP_REFED = 25,
821 QP_DEREFED = 26,
822 CM_ID_REFED = 27,
823 CM_ID_DEREFED = 28
824 };
825
826 struct c4iw_ep_common {
827 TAILQ_ENTRY(c4iw_ep_common) entry; /* Work queue attachment */
828 struct iw_cm_id *cm_id;
829 struct c4iw_qp *qp;
830 struct c4iw_dev *dev;
831 enum c4iw_ep_state state;
832 struct kref kref;
833 struct mutex mutex;
834 struct sockaddr_storage local_addr;
835 struct sockaddr_storage remote_addr;
836 struct c4iw_wr_wait wr_wait;
837 unsigned long flags;
838 unsigned long history;
839 int rpl_err;
840 int rpl_done;
841 struct thread *thread;
842 struct socket *so;
843 int ep_events;
844 };
845
846 struct c4iw_listen_ep {
847 struct c4iw_ep_common com;
848 unsigned int stid;
849 int backlog;
850 struct list_head listen_ep_list; /* list of all listener ep's bound
851 to one port address */
852 };
853
854 struct c4iw_ep {
855 struct c4iw_ep_common com;
856 struct c4iw_listen_ep *parent_ep;
857 struct timer_list timer;
858 unsigned int atid;
859 u32 hwtid;
860 u32 snd_seq;
861 u32 rcv_seq;
862 struct l2t_entry *l2t;
863 struct dst_entry *dst;
864 struct c4iw_mpa_attributes mpa_attr;
865 u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA];
866 unsigned int mpa_pkt_len;
867 u32 ird;
868 u32 ord;
869 u32 tx_chan;
870 u32 mtu;
871 u16 mss;
872 u16 plen;
873 u16 rss_qid;
874 u16 txq_idx;
875 u16 ctrlq_idx;
876 u8 tos;
877 u8 retry_with_mpa_v1;
878 u8 tried_with_mpa_v1;
879 };
880
to_ep(struct iw_cm_id * cm_id)881 static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id)
882 {
883 return cm_id->provider_data;
884 }
885
to_listen_ep(struct iw_cm_id * cm_id)886 static inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id)
887 {
888 return cm_id->provider_data;
889 }
890
compute_wscale(int win)891 static inline int compute_wscale(int win)
892 {
893 int wscale = 0;
894
895 while (wscale < 14 && (65535<<wscale) < win)
896 wscale++;
897 return wscale;
898 }
899
900 u32 c4iw_id_alloc(struct c4iw_id_table *alloc);
901 void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj);
902 int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,
903 u32 reserved, u32 flags);
904 void c4iw_id_table_free(struct c4iw_id_table *alloc);
905
906 typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct mbuf *m);
907
908 int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
909 struct l2t_entry *l2t);
910 u32 c4iw_get_resource(struct c4iw_id_table *id_table);
911 void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry);
912 int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt, u32 nr_pdid);
913 int c4iw_init_ctrl_qp(struct c4iw_rdev *rdev);
914 int c4iw_pblpool_create(struct c4iw_rdev *rdev);
915 int c4iw_rqtpool_create(struct c4iw_rdev *rdev);
916 void c4iw_pblpool_destroy(struct c4iw_rdev *rdev);
917 void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev);
918 void c4iw_destroy_resource(struct c4iw_resource *rscp);
919 int c4iw_destroy_ctrl_qp(struct c4iw_rdev *rdev);
920 int c4iw_register_device(struct c4iw_dev *dev);
921 void c4iw_unregister_device(struct c4iw_dev *dev);
922 int __init c4iw_cm_init(void);
923 void __exit c4iw_cm_term(void);
924 void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,
925 struct c4iw_dev_ucontext *uctx);
926 void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev,
927 struct c4iw_dev_ucontext *uctx);
928 int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
929 int c4iw_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
930 const struct ib_send_wr **bad_wr);
931 int c4iw_post_receive(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
932 const struct ib_recv_wr **bad_wr);
933 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
934 int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog);
935 int c4iw_destroy_listen(struct iw_cm_id *cm_id);
936 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
937 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
938 void c4iw_qp_add_ref(struct ib_qp *qp);
939 void c4iw_qp_rem_ref(struct ib_qp *qp);
940 struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
941 u32 max_num_sg);
942 int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
943 int sg_nents, unsigned int *sg_offset);
944 int c4iw_dealloc_mw(struct ib_mw *mw);
945 struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
946 struct ib_udata *udata);
947 struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64
948 virt, int acc, struct ib_udata *udata);
949 struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
950 int c4iw_dereg_mr(struct ib_mr *ib_mr);
951 void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey);
952 int c4iw_destroy_cq(struct ib_cq *ib_cq);
953 struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
954 const struct ib_cq_init_attr *attr,
955 struct ib_ucontext *ib_context,
956 struct ib_udata *udata);
957 int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
958 int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
959 int c4iw_destroy_qp(struct ib_qp *ib_qp);
960 struct ib_qp *c4iw_create_qp(struct ib_pd *pd,
961 struct ib_qp_init_attr *attrs,
962 struct ib_udata *udata);
963 int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
964 int attr_mask, struct ib_udata *udata);
965 int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
966 int attr_mask, struct ib_qp_init_attr *init_attr);
967 struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn);
968 u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size);
969 void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
970 u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size);
971 void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
972 int c4iw_ofld_send(struct c4iw_rdev *rdev, struct mbuf *m);
973 void c4iw_flush_hw_cq(struct c4iw_cq *cq);
974 void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
975 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
976 int __c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
977 int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);
978 int c4iw_flush_sq(struct c4iw_qp *qhp);
979 int c4iw_ev_handler(struct sge_iq *, const struct rsp_ctrl *);
980 u16 c4iw_rqes_posted(struct c4iw_qp *qhp);
981 int c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe);
982 u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
983 void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,
984 struct c4iw_dev_ucontext *uctx);
985 u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
986 void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
987 struct c4iw_dev_ucontext *uctx);
988 void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe);
989 #endif
990