1 /*
2 * Copyright (c) 2018-2019 Cavium, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 /*
29 * File: qlnxr_verbs.c
30 */
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include "qlnxr_def.h"
35 #include "rdma_common.h"
36 #include "qlnxr_roce.h"
37 #include "qlnxr_cm.h"
38
39 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo))
40
41 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \
42 do { \
43 (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\
44 (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\
45 } while (0)
46
47 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
48 do { \
49 TYPEPTR_ADDR_SET(sge, addr, vaddr); \
50 (sge)->length = cpu_to_le32(vlength); \
51 (sge)->flags = cpu_to_le32(vflags); \
52 } while (0)
53
54 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
55 do { \
56 TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \
57 (hdr)->num_sges = num_sge; \
58 } while (0)
59
60 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
61 do { \
62 TYPEPTR_ADDR_SET(sge, addr, vaddr); \
63 (sge)->length = cpu_to_le32(vlength); \
64 (sge)->l_key = cpu_to_le32(vlkey); \
65 } while (0)
66
67 #define NIPQUAD(addr) \
68 ((unsigned char *)&addr)[0], \
69 ((unsigned char *)&addr)[1], \
70 ((unsigned char *)&addr)[2], \
71 ((unsigned char *)&addr)[3]
72
73 static int
74 qlnxr_check_srq_params(struct ib_pd *ibpd,
75 struct qlnxr_dev *dev,
76 struct ib_srq_init_attr *attrs);
77
78 static int
79 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
80 struct qlnxr_srq *srq,
81 struct qlnxr_create_srq_ureq *ureq,
82 int access, int dmasync);
83
84 static int
85 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
86 struct qlnxr_dev *dev,
87 struct ib_srq_init_attr *init_attr);
88
89 static int
90 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
91 struct qlnxr_srq *srq,
92 struct ib_udata *udata);
93
94 static void
95 qlnxr_free_srq_user_params(struct qlnxr_srq *srq);
96
97 static void
98 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq);
99
100 static u32
101 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq);
102
103 int
qlnxr_iw_query_gid(struct ib_device * ibdev,u8 port,int index,union ib_gid * sgid)104 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index,
105 union ib_gid *sgid)
106 {
107 struct qlnxr_dev *dev;
108 qlnx_host_t *ha;
109
110 dev = get_qlnxr_dev(ibdev);
111 ha = dev->ha;
112
113 QL_DPRINT12(ha, "enter\n");
114
115 memset(sgid->raw, 0, sizeof(sgid->raw));
116
117 memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac));
118
119 QL_DPRINT12(ha, "exit\n");
120
121 return 0;
122 }
123
124 int
qlnxr_query_gid(struct ib_device * ibdev,u8 port,int index,union ib_gid * sgid)125 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index,
126 union ib_gid *sgid)
127 {
128 struct qlnxr_dev *dev;
129 qlnx_host_t *ha;
130
131 dev = get_qlnxr_dev(ibdev);
132 ha = dev->ha;
133 QL_DPRINT12(ha, "enter index: %d\n", index);
134 #if 0
135 int ret = 0;
136 /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */
137 //if (!rdma_cap_roce_gid_table(ibdev, port)) {
138 if (!(rdma_protocol_roce(ibdev, port) &&
139 ibdev->add_gid && ibdev->del_gid)) {
140 QL_DPRINT11(ha, "acquire gid failed\n");
141 return -ENODEV;
142 }
143
144 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
145 if (ret == -EAGAIN) {
146 memcpy(sgid, &zgid, sizeof(*sgid));
147 return 0;
148 }
149 #endif
150 if ((index >= QLNXR_MAX_SGID) || (index < 0)) {
151 QL_DPRINT12(ha, "invalid gid index %d\n", index);
152 memset(sgid, 0, sizeof(*sgid));
153 return -EINVAL;
154 }
155 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));
156
157 QL_DPRINT12(ha, "exit : %p\n", sgid);
158
159 return 0;
160 }
161
162 struct ib_srq *
qlnxr_create_srq(struct ib_pd * ibpd,struct ib_srq_init_attr * init_attr,struct ib_udata * udata)163 qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
164 struct ib_udata *udata)
165 {
166 struct qlnxr_dev *dev;
167 qlnx_host_t *ha;
168 struct ecore_rdma_destroy_srq_in_params destroy_in_params;
169 struct ecore_rdma_create_srq_out_params out_params;
170 struct ecore_rdma_create_srq_in_params in_params;
171 u64 pbl_base_addr, phy_prod_pair_addr;
172 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
173 struct ib_ucontext *ib_ctx = NULL;
174 struct qlnxr_srq_hwq_info *hw_srq;
175 struct qlnxr_ucontext *ctx = NULL;
176 struct qlnxr_create_srq_ureq ureq;
177 u32 page_cnt, page_size;
178 struct qlnxr_srq *srq;
179 int ret = 0;
180
181 dev = get_qlnxr_dev((ibpd->device));
182 ha = dev->ha;
183
184 QL_DPRINT12(ha, "enter\n");
185
186 ret = qlnxr_check_srq_params(ibpd, dev, init_attr);
187
188 srq = kzalloc(sizeof(*srq), GFP_KERNEL);
189 if (!srq) {
190 QL_DPRINT11(ha, "cannot allocate memory for srq\n");
191 return NULL; //@@@ : TODO what to return here?
192 }
193
194 srq->dev = dev;
195 hw_srq = &srq->hw_srq;
196 spin_lock_init(&srq->lock);
197 memset(&in_params, 0, sizeof(in_params));
198
199 if (udata && ibpd->uobject && ibpd->uobject->context) {
200 ib_ctx = ibpd->uobject->context;
201 ctx = get_qlnxr_ucontext(ib_ctx);
202
203 memset(&ureq, 0, sizeof(ureq));
204 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
205 udata->inlen))) {
206 QL_DPRINT11(ha, "problem"
207 " copying data from user space\n");
208 goto err0;
209 }
210
211 ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
212 if (ret)
213 goto err0;
214
215 page_cnt = srq->usrq.pbl_info.num_pbes;
216 pbl_base_addr = srq->usrq.pbl_tbl->pa;
217 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
218 // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT
219 // page_size = BIT(srq->usrq.umem->page_shift);
220 // else
221 page_size = srq->usrq.umem->page_size;
222 } else {
223 struct ecore_chain *pbl;
224 ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr);
225 if (ret)
226 goto err0;
227 pbl = &hw_srq->pbl;
228
229 page_cnt = ecore_chain_get_page_cnt(pbl);
230 pbl_base_addr = ecore_chain_get_pbl_phys(pbl);
231 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
232 page_size = pbl->elem_per_page << 4;
233 }
234
235 in_params.pd_id = pd->pd_id;
236 in_params.pbl_base_addr = pbl_base_addr;
237 in_params.prod_pair_addr = phy_prod_pair_addr;
238 in_params.num_pages = page_cnt;
239 in_params.page_size = page_size;
240
241 ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
242 if (ret)
243 goto err1;
244
245 srq->srq_id = out_params.srq_id;
246
247 if (udata) {
248 ret = qlnxr_copy_srq_uresp(dev, srq, udata);
249 if (ret)
250 goto err2;
251 }
252
253 QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id);
254 return &srq->ibsrq;
255 err2:
256 memset(&in_params, 0, sizeof(in_params));
257 destroy_in_params.srq_id = srq->srq_id;
258 ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
259
260 err1:
261 if (udata)
262 qlnxr_free_srq_user_params(srq);
263 else
264 qlnxr_free_srq_kernel_params(srq);
265
266 err0:
267 kfree(srq);
268 return ERR_PTR(-EFAULT);
269 }
270
271 int
qlnxr_destroy_srq(struct ib_srq * ibsrq)272 qlnxr_destroy_srq(struct ib_srq *ibsrq)
273 {
274 struct qlnxr_dev *dev;
275 struct qlnxr_srq *srq;
276 qlnx_host_t *ha;
277 struct ecore_rdma_destroy_srq_in_params in_params;
278
279 srq = get_qlnxr_srq(ibsrq);
280 dev = srq->dev;
281 ha = dev->ha;
282
283 memset(&in_params, 0, sizeof(in_params));
284 in_params.srq_id = srq->srq_id;
285
286 ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params);
287
288 if (ibsrq->pd->uobject && ibsrq->pd->uobject->context)
289 qlnxr_free_srq_user_params(srq);
290 else
291 qlnxr_free_srq_kernel_params(srq);
292
293 QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id);
294 kfree(srq);
295 return 0;
296 }
297
298 int
qlnxr_modify_srq(struct ib_srq * ibsrq,struct ib_srq_attr * attr,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)299 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
300 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
301 {
302 struct qlnxr_dev *dev;
303 struct qlnxr_srq *srq;
304 qlnx_host_t *ha;
305 struct ecore_rdma_modify_srq_in_params in_params;
306 int ret = 0;
307
308 srq = get_qlnxr_srq(ibsrq);
309 dev = srq->dev;
310 ha = dev->ha;
311
312 QL_DPRINT12(ha, "enter\n");
313 if (attr_mask & IB_SRQ_MAX_WR) {
314 QL_DPRINT12(ha, "invalid attribute mask=0x%x"
315 " specified for %p\n", attr_mask, srq);
316 return -EINVAL;
317 }
318
319 if (attr_mask & IB_SRQ_LIMIT) {
320 if (attr->srq_limit >= srq->hw_srq.max_wr) {
321 QL_DPRINT12(ha, "invalid srq_limit=0x%x"
322 " (max_srq_limit = 0x%x)\n",
323 attr->srq_limit, srq->hw_srq.max_wr);
324 return -EINVAL;
325 }
326 memset(&in_params, 0, sizeof(in_params));
327 in_params.srq_id = srq->srq_id;
328 in_params.wqe_limit = attr->srq_limit;
329 ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params);
330 if (ret)
331 return ret;
332 }
333
334 QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id);
335 return 0;
336 }
337
338 int
qlnxr_query_srq(struct ib_srq * ibsrq,struct ib_srq_attr * srq_attr)339 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
340 {
341 struct qlnxr_dev *dev;
342 struct qlnxr_srq *srq;
343 qlnx_host_t *ha;
344 struct ecore_rdma_device *qattr;
345 srq = get_qlnxr_srq(ibsrq);
346 dev = srq->dev;
347 ha = dev->ha;
348 //qattr = &dev->attr;
349 qattr = ecore_rdma_query_device(dev->rdma_ctx);
350 QL_DPRINT12(ha, "enter\n");
351
352 if (!dev->rdma_ctx) {
353 QL_DPRINT12(ha, "called with invalid params"
354 " rdma_ctx is NULL\n");
355 return -EINVAL;
356 }
357
358 srq_attr->srq_limit = qattr->max_srq;
359 srq_attr->max_wr = qattr->max_srq_wr;
360 srq_attr->max_sge = qattr->max_sge;
361
362 QL_DPRINT12(ha, "exit\n");
363 return 0;
364 }
365
366 /* Increment srq wr producer by one */
367 static
qlnxr_inc_srq_wr_prod(struct qlnxr_srq_hwq_info * info)368 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info)
369 {
370 info->wr_prod_cnt++;
371 }
372
373 /* Increment srq wr consumer by one */
374 static
qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info * info)375 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info)
376 {
377 info->wr_cons_cnt++;
378 }
379
380 /* get_port_immutable verb is not available in FreeBSD */
381 #if 0
382 int
qlnxr_roce_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)383 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
384 struct ib_port_immutable *immutable)
385 {
386 struct qlnxr_dev *dev;
387 qlnx_host_t *ha;
388 dev = get_qlnxr_dev(ibdev);
389 ha = dev->ha;
390
391 QL_DPRINT12(ha, "entered but not implemented!!!\n");
392 }
393 #endif
394
395 int
qlnxr_post_srq_recv(struct ib_srq * ibsrq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)396 qlnxr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
397 const struct ib_recv_wr **bad_wr)
398 {
399 struct qlnxr_dev *dev;
400 struct qlnxr_srq *srq;
401 qlnx_host_t *ha;
402 struct qlnxr_srq_hwq_info *hw_srq;
403 struct ecore_chain *pbl;
404 unsigned long flags;
405 int status = 0;
406 u32 num_sge, offset;
407
408 srq = get_qlnxr_srq(ibsrq);
409 dev = srq->dev;
410 ha = dev->ha;
411 hw_srq = &srq->hw_srq;
412
413 QL_DPRINT12(ha, "enter\n");
414 spin_lock_irqsave(&srq->lock, flags);
415
416 pbl = &srq->hw_srq.pbl;
417 while (wr) {
418 struct rdma_srq_wqe_header *hdr;
419 int i;
420
421 if (!qlnxr_srq_elem_left(hw_srq) ||
422 wr->num_sge > srq->hw_srq.max_sges) {
423 QL_DPRINT11(ha, "WR cannot be posted"
424 " (%d, %d) || (%d > %d)\n",
425 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
426 wr->num_sge, srq->hw_srq.max_sges);
427 status = -ENOMEM;
428 *bad_wr = wr;
429 break;
430 }
431
432 hdr = ecore_chain_produce(pbl);
433 num_sge = wr->num_sge;
434 /* Set number of sge and WR id in header */
435 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
436
437 /* PBL is maintained in case of WR granularity.
438 * So increment WR producer in case we post a WR.
439 */
440 qlnxr_inc_srq_wr_prod(hw_srq);
441 hw_srq->wqe_prod++;
442 hw_srq->sge_prod++;
443
444 QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n",
445 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
446
447 for (i = 0; i < wr->num_sge; i++) {
448 struct rdma_srq_sge *srq_sge =
449 ecore_chain_produce(pbl);
450 /* Set SGE length, lkey and address */
451 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
452 wr->sg_list[i].length, wr->sg_list[i].lkey);
453
454 QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n",
455 i, srq_sge->length, srq_sge->l_key,
456 srq_sge->addr.hi, srq_sge->addr.lo);
457 hw_srq->sge_prod++;
458 }
459 wmb();
460 /*
461 * SRQ prod is 8 bytes. Need to update SGE prod in index
462 * in first 4 bytes and need to update WQE prod in next
463 * 4 bytes.
464 */
465 *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod;
466 offset = offsetof(struct rdma_srq_producers, wqe_prod);
467 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
468 hw_srq->wqe_prod;
469 /* Flush prod after updating it */
470 wmb();
471 wr = wr->next;
472 }
473
474 QL_DPRINT12(ha, "Elements in SRQ: %d\n",
475 ecore_chain_get_elem_left(pbl));
476
477 spin_unlock_irqrestore(&srq->lock, flags);
478 QL_DPRINT12(ha, "exit\n");
479 return status;
480 }
481
482 int
483 #if __FreeBSD_version < 1102000
qlnxr_query_device(struct ib_device * ibdev,struct ib_device_attr * attr)484 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
485 #else
486 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
487 struct ib_udata *udata)
488 #endif /* #if __FreeBSD_version < 1102000 */
489
490 {
491 struct qlnxr_dev *dev;
492 struct ecore_rdma_device *qattr;
493 qlnx_host_t *ha;
494
495 dev = get_qlnxr_dev(ibdev);
496 ha = dev->ha;
497
498 QL_DPRINT12(ha, "enter\n");
499
500 #if __FreeBSD_version > 1102000
501 if (udata->inlen || udata->outlen)
502 return -EINVAL;
503 #endif /* #if __FreeBSD_version > 1102000 */
504
505 if (dev->rdma_ctx == NULL) {
506 return -EINVAL;
507 }
508
509 qattr = ecore_rdma_query_device(dev->rdma_ctx);
510
511 memset(attr, 0, sizeof *attr);
512
513 attr->fw_ver = qattr->fw_ver;
514 attr->sys_image_guid = qattr->sys_image_guid;
515 attr->max_mr_size = qattr->max_mr_size;
516 attr->page_size_cap = qattr->page_size_caps;
517 attr->vendor_id = qattr->vendor_id;
518 attr->vendor_part_id = qattr->vendor_part_id;
519 attr->hw_ver = qattr->hw_ver;
520 attr->max_qp = qattr->max_qp;
521 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
522 IB_DEVICE_RC_RNR_NAK_GEN |
523 IB_DEVICE_LOCAL_DMA_LKEY |
524 IB_DEVICE_MEM_MGT_EXTENSIONS;
525
526 attr->max_sge = qattr->max_sge;
527 attr->max_sge_rd = qattr->max_sge;
528 attr->max_cq = qattr->max_cq;
529 attr->max_cqe = qattr->max_cqe;
530 attr->max_mr = qattr->max_mr;
531 attr->max_mw = qattr->max_mw;
532 attr->max_pd = qattr->max_pd;
533 attr->atomic_cap = dev->atomic_cap;
534 attr->max_fmr = qattr->max_fmr;
535 attr->max_map_per_fmr = 16; /* TBD: FMR */
536
537 /* There is an implicit assumption in some of the ib_xxx apps that the
538 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in
539 * communication the qp_rd_atom is passed to the other side and used as
540 * init_rd_atom without check device capabilities for init_rd_atom.
541 * for this reason, we set the qp_rd_atom to be the minimum between the
542 * two...There is an additional assumption in mlx4 driver that the
543 * values are power of two, fls is performed on the value - 1, which
544 * in fact gives a larger power of two for values which are not a power
545 * of two. This should be fixed in mlx4 driver, but until then ->
546 * we provide a value that is a power of two in our code.
547 */
548 attr->max_qp_init_rd_atom =
549 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
550 attr->max_qp_rd_atom =
551 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
552 attr->max_qp_init_rd_atom);
553
554 attr->max_srq = qattr->max_srq;
555 attr->max_srq_sge = qattr->max_srq_sge;
556 attr->max_srq_wr = qattr->max_srq_wr;
557
558 /* TODO: R&D to more properly configure the following */
559 attr->local_ca_ack_delay = qattr->dev_ack_delay;
560 attr->max_fast_reg_page_list_len = qattr->max_mr/8;
561 attr->max_pkeys = QLNXR_ROCE_PKEY_MAX;
562 attr->max_ah = qattr->max_ah;
563
564 QL_DPRINT12(ha, "exit\n");
565 return 0;
566 }
567
568 static inline void
get_link_speed_and_width(int speed,uint8_t * ib_speed,uint8_t * ib_width)569 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width)
570 {
571 switch (speed) {
572 case 1000:
573 *ib_speed = IB_SPEED_SDR;
574 *ib_width = IB_WIDTH_1X;
575 break;
576 case 10000:
577 *ib_speed = IB_SPEED_QDR;
578 *ib_width = IB_WIDTH_1X;
579 break;
580
581 case 20000:
582 *ib_speed = IB_SPEED_DDR;
583 *ib_width = IB_WIDTH_4X;
584 break;
585
586 case 25000:
587 *ib_speed = IB_SPEED_EDR;
588 *ib_width = IB_WIDTH_1X;
589 break;
590
591 case 40000:
592 *ib_speed = IB_SPEED_QDR;
593 *ib_width = IB_WIDTH_4X;
594 break;
595
596 case 50000:
597 *ib_speed = IB_SPEED_QDR;
598 *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50...
599 break;
600
601 case 100000:
602 *ib_speed = IB_SPEED_EDR;
603 *ib_width = IB_WIDTH_4X;
604 break;
605
606 default:
607 /* Unsupported */
608 *ib_speed = IB_SPEED_SDR;
609 *ib_width = IB_WIDTH_1X;
610 }
611 return;
612 }
613
614 int
qlnxr_query_port(struct ib_device * ibdev,uint8_t port,struct ib_port_attr * attr)615 qlnxr_query_port(struct ib_device *ibdev, uint8_t port,
616 struct ib_port_attr *attr)
617 {
618 struct qlnxr_dev *dev;
619 struct ecore_rdma_port *rdma_port;
620 qlnx_host_t *ha;
621
622 dev = get_qlnxr_dev(ibdev);
623 ha = dev->ha;
624
625 QL_DPRINT12(ha, "enter\n");
626
627 if (port > 1) {
628 QL_DPRINT12(ha, "port [%d] > 1 \n", port);
629 return -EINVAL;
630 }
631
632 if (dev->rdma_ctx == NULL) {
633 QL_DPRINT12(ha, "rdma_ctx == NULL\n");
634 return -EINVAL;
635 }
636
637 rdma_port = ecore_rdma_query_port(dev->rdma_ctx);
638 memset(attr, 0, sizeof *attr);
639
640 if (rdma_port->port_state == ECORE_RDMA_PORT_UP) {
641 attr->state = IB_PORT_ACTIVE;
642 attr->phys_state = 5;
643 } else {
644 attr->state = IB_PORT_DOWN;
645 attr->phys_state = 3;
646 }
647
648 attr->max_mtu = IB_MTU_4096;
649 attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu);
650 attr->lid = 0;
651 attr->lmc = 0;
652 attr->sm_lid = 0;
653 attr->sm_sl = 0;
654 attr->port_cap_flags = 0;
655
656 if (QLNX_IS_IWARP(dev)) {
657 attr->gid_tbl_len = 1;
658 attr->pkey_tbl_len = 1;
659 } else {
660 attr->gid_tbl_len = QLNXR_MAX_SGID;
661 attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN;
662 }
663
664 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
665 attr->qkey_viol_cntr = 0;
666
667 get_link_speed_and_width(rdma_port->link_speed,
668 &attr->active_speed, &attr->active_width);
669
670 attr->max_msg_sz = rdma_port->max_msg_size;
671 attr->max_vl_num = 4; /* TODO -> figure this one out... */
672
673 QL_DPRINT12(ha, "state = %d phys_state = %d "
674 " link_speed = %d active_speed = %d active_width = %d"
675 " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d"
676 " max_msg_sz = 0x%x max_vl_num = 0x%x \n",
677 attr->state, attr->phys_state,
678 rdma_port->link_speed, attr->active_speed,
679 attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len,
680 attr->max_msg_sz, attr->max_vl_num);
681
682 QL_DPRINT12(ha, "exit\n");
683 return 0;
684 }
685
686 int
qlnxr_modify_port(struct ib_device * ibdev,uint8_t port,int mask,struct ib_port_modify * props)687 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask,
688 struct ib_port_modify *props)
689 {
690 struct qlnxr_dev *dev;
691 qlnx_host_t *ha;
692
693 dev = get_qlnxr_dev(ibdev);
694 ha = dev->ha;
695
696 QL_DPRINT12(ha, "enter\n");
697
698 if (port > 1) {
699 QL_DPRINT12(ha, "port (%d) > 1\n", port);
700 return -EINVAL;
701 }
702
703 QL_DPRINT12(ha, "exit\n");
704 return 0;
705 }
706
707 enum rdma_link_layer
qlnxr_link_layer(struct ib_device * ibdev,uint8_t port_num)708 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num)
709 {
710 struct qlnxr_dev *dev;
711 qlnx_host_t *ha;
712
713 dev = get_qlnxr_dev(ibdev);
714 ha = dev->ha;
715
716 QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num);
717
718 return IB_LINK_LAYER_ETHERNET;
719 }
720
721 struct ib_pd *
qlnxr_alloc_pd(struct ib_device * ibdev,struct ib_ucontext * context,struct ib_udata * udata)722 qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
723 struct ib_udata *udata)
724 {
725 struct qlnxr_pd *pd = NULL;
726 u16 pd_id;
727 int rc;
728 struct qlnxr_dev *dev;
729 qlnx_host_t *ha;
730
731 dev = get_qlnxr_dev(ibdev);
732 ha = dev->ha;
733
734 QL_DPRINT12(ha, "ibdev = %p context = %p"
735 " udata = %p enter\n", ibdev, context, udata);
736
737 if (dev->rdma_ctx == NULL) {
738 QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n");
739 rc = -1;
740 goto err;
741 }
742
743 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
744 if (!pd) {
745 rc = -ENOMEM;
746 QL_DPRINT11(ha, "kzalloc(pd) = NULL\n");
747 goto err;
748 }
749
750 rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id);
751 if (rc) {
752 QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n");
753 goto err;
754 }
755
756 pd->pd_id = pd_id;
757
758 if (udata && context) {
759 rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id));
760 if (rc) {
761 QL_DPRINT11(ha, "ib_copy_to_udata failed\n");
762 ecore_rdma_free_pd(dev->rdma_ctx, pd_id);
763 goto err;
764 }
765
766 pd->uctx = get_qlnxr_ucontext(context);
767 pd->uctx->pd = pd;
768 }
769
770 atomic_add_rel_32(&dev->pd_count, 1);
771 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
772 pd, pd_id, dev->pd_count);
773
774 return &pd->ibpd;
775
776 err:
777 kfree(pd);
778 QL_DPRINT12(ha, "exit -1\n");
779 return ERR_PTR(rc);
780 }
781
782 int
qlnxr_dealloc_pd(struct ib_pd * ibpd)783 qlnxr_dealloc_pd(struct ib_pd *ibpd)
784 {
785 struct qlnxr_pd *pd;
786 struct qlnxr_dev *dev;
787 qlnx_host_t *ha;
788
789 pd = get_qlnxr_pd(ibpd);
790 dev = get_qlnxr_dev((ibpd->device));
791 ha = dev->ha;
792
793 QL_DPRINT12(ha, "enter\n");
794
795 if (pd == NULL) {
796 QL_DPRINT11(ha, "pd = NULL\n");
797 } else {
798 ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id);
799 kfree(pd);
800 atomic_subtract_rel_32(&dev->pd_count, 1);
801 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
802 pd, pd->pd_id, dev->pd_count);
803 }
804
805 QL_DPRINT12(ha, "exit\n");
806 return 0;
807 }
808
809 #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge)
810 #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */
811 /* Should be part of HSI */
812 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */
813 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
814
815 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *);
816 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *);
817
818 int
qlnxr_query_pkey(struct ib_device * ibdev,u8 port,u16 index,u16 * pkey)819 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
820 {
821 struct qlnxr_dev *dev;
822 qlnx_host_t *ha;
823
824 dev = get_qlnxr_dev(ibdev);
825 ha = dev->ha;
826
827 QL_DPRINT12(ha, "enter index = 0x%x\n", index);
828
829 if (index > QLNXR_ROCE_PKEY_TABLE_LEN)
830 return -EINVAL;
831
832 *pkey = QLNXR_ROCE_PKEY_DEFAULT;
833
834 QL_DPRINT12(ha, "exit\n");
835 return 0;
836 }
837
838 static inline bool
qlnxr_get_vlan_id_qp(qlnx_host_t * ha,struct ib_qp_attr * attr,int attr_mask,u16 * vlan_id)839 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask,
840 u16 *vlan_id)
841 {
842 bool ret = false;
843
844 QL_DPRINT12(ha, "enter \n");
845
846 *vlan_id = 0;
847
848 #if __FreeBSD_version >= 1100000
849 u16 tmp_vlan_id;
850
851 #if __FreeBSD_version >= 1102000
852 union ib_gid *dgid;
853
854 dgid = &attr->ah_attr.grh.dgid;
855 tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12];
856
857 if (!(tmp_vlan_id & ~EVL_VLID_MASK)) {
858 *vlan_id = tmp_vlan_id;
859 ret = true;
860 }
861 #else
862 tmp_vlan_id = attr->vlan_id;
863
864 if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) {
865 *vlan_id = tmp_vlan_id;
866 ret = true;
867 }
868
869 #endif /* #if __FreeBSD_version > 1102000 */
870
871 #else
872 ret = true;
873
874 #endif /* #if __FreeBSD_version >= 1100000 */
875
876 QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret);
877
878 return (ret);
879 }
880
881 static inline void
get_gid_info(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct qlnxr_dev * dev,struct qlnxr_qp * qp,struct ecore_rdma_modify_qp_in_params * qp_params)882 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr,
883 int attr_mask,
884 struct qlnxr_dev *dev,
885 struct qlnxr_qp *qp,
886 struct ecore_rdma_modify_qp_in_params *qp_params)
887 {
888 int i;
889 qlnx_host_t *ha;
890
891 ha = dev->ha;
892
893 QL_DPRINT12(ha, "enter\n");
894
895 memcpy(&qp_params->sgid.bytes[0],
896 &dev->sgid_tbl[qp->sgid_idx].raw[0],
897 sizeof(qp_params->sgid.bytes));
898 memcpy(&qp_params->dgid.bytes[0],
899 &attr->ah_attr.grh.dgid.raw[0],
900 sizeof(qp_params->dgid));
901
902 qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id);
903
904 for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) {
905 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
906 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
907 }
908
909 QL_DPRINT12(ha, "exit\n");
910 return;
911 }
912
913 static int
qlnxr_add_mmap(struct qlnxr_ucontext * uctx,u64 phy_addr,unsigned long len)914 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
915 {
916 struct qlnxr_mm *mm;
917 qlnx_host_t *ha;
918
919 ha = uctx->dev->ha;
920
921 QL_DPRINT12(ha, "enter\n");
922
923 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
924 if (mm == NULL) {
925 QL_DPRINT11(ha, "mm = NULL\n");
926 return -ENOMEM;
927 }
928
929 mm->key.phy_addr = phy_addr;
930
931 /* This function might be called with a length which is not a multiple
932 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
933 * forces this granularity by increasing the requested size if needed.
934 * When qedr_mmap is called, it will search the list with the updated
935 * length as a key. To prevent search failures, the length is rounded up
936 * in advance to PAGE_SIZE.
937 */
938 mm->key.len = roundup(len, PAGE_SIZE);
939 INIT_LIST_HEAD(&mm->entry);
940
941 mutex_lock(&uctx->mm_list_lock);
942 list_add(&mm->entry, &uctx->mm_head);
943 mutex_unlock(&uctx->mm_list_lock);
944
945 QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
946 (unsigned long long)mm->key.phy_addr,
947 (unsigned long)mm->key.len, uctx);
948
949 return 0;
950 }
951
952 static bool
qlnxr_search_mmap(struct qlnxr_ucontext * uctx,u64 phy_addr,unsigned long len)953 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
954 {
955 bool found = false;
956 struct qlnxr_mm *mm;
957 qlnx_host_t *ha;
958
959 ha = uctx->dev->ha;
960
961 QL_DPRINT12(ha, "enter\n");
962
963 mutex_lock(&uctx->mm_list_lock);
964 list_for_each_entry(mm, &uctx->mm_head, entry) {
965 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
966 continue;
967
968 found = true;
969 break;
970 }
971 mutex_unlock(&uctx->mm_list_lock);
972
973 QL_DPRINT12(ha,
974 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n",
975 mm->key.phy_addr, mm->key.len, uctx, found);
976
977 return found;
978 }
979
980 struct
qlnxr_alloc_ucontext(struct ib_device * ibdev,struct ib_udata * udata)981 ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
982 struct ib_udata *udata)
983 {
984 int rc;
985 struct qlnxr_ucontext *ctx;
986 struct qlnxr_alloc_ucontext_resp uresp;
987 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
988 qlnx_host_t *ha = dev->ha;
989 struct ecore_rdma_add_user_out_params oparams;
990
991 if (!udata) {
992 return ERR_PTR(-EFAULT);
993 }
994
995 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
996 if (!ctx)
997 return ERR_PTR(-ENOMEM);
998
999 rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams);
1000 if (rc) {
1001 QL_DPRINT12(ha,
1002 "Failed to allocate a DPI for a new RoCE application "
1003 ",rc = %d. To overcome this, consider to increase "
1004 "the number of DPIs, increase the doorbell BAR size "
1005 "or just close unnecessary RoCE applications. In "
1006 "order to increase the number of DPIs consult the "
1007 "README\n", rc);
1008 goto err;
1009 }
1010
1011 ctx->dpi = oparams.dpi;
1012 ctx->dpi_addr = oparams.dpi_addr;
1013 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
1014 ctx->dpi_size = oparams.dpi_size;
1015 INIT_LIST_HEAD(&ctx->mm_head);
1016 mutex_init(&ctx->mm_list_lock);
1017
1018 memset(&uresp, 0, sizeof(uresp));
1019 uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled)
1020 < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out
1021 uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled)
1022 < udata->outlen ? 1 : 0; //TODO: figure this out
1023 uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count)
1024 < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out
1025 uresp.db_pa = ctx->dpi_phys_addr;
1026 uresp.db_size = ctx->dpi_size;
1027 uresp.max_send_wr = dev->attr.max_sqe;
1028 uresp.max_recv_wr = dev->attr.max_rqe;
1029 uresp.max_srq_wr = dev->attr.max_srq_wr;
1030 uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
1031 uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
1032 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
1033 uresp.max_cqes = QLNXR_MAX_CQES;
1034
1035 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1036 if (rc)
1037 goto err;
1038
1039 ctx->dev = dev;
1040
1041 rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
1042 if (rc)
1043 goto err;
1044 QL_DPRINT12(ha, "Allocated user context %p\n",
1045 &ctx->ibucontext);
1046
1047 return &ctx->ibucontext;
1048 err:
1049 kfree(ctx);
1050 return ERR_PTR(rc);
1051 }
1052
1053 int
qlnxr_dealloc_ucontext(struct ib_ucontext * ibctx)1054 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx)
1055 {
1056 struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx);
1057 struct qlnxr_dev *dev = uctx->dev;
1058 qlnx_host_t *ha = dev->ha;
1059 struct qlnxr_mm *mm, *tmp;
1060 int status = 0;
1061
1062 QL_DPRINT12(ha, "Deallocating user context %p\n",
1063 uctx);
1064
1065 if (dev) {
1066 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
1067 }
1068
1069 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
1070 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for"
1071 " ctx=%p\n",
1072 mm->key.phy_addr, mm->key.len, uctx);
1073 list_del(&mm->entry);
1074 kfree(mm);
1075 }
1076 kfree(uctx);
1077 return status;
1078 }
1079
1080 int
qlnxr_mmap(struct ib_ucontext * context,struct vm_area_struct * vma)1081 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
1082 {
1083 struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context);
1084 struct qlnxr_dev *dev = get_qlnxr_dev((context->device));
1085 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
1086 u64 unmapped_db;
1087 unsigned long len = (vma->vm_end - vma->vm_start);
1088 int rc = 0;
1089 bool found;
1090 qlnx_host_t *ha;
1091
1092 ha = dev->ha;
1093
1094 #if __FreeBSD_version > 1102000
1095 unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
1096 #else
1097 unmapped_db = dev->db_phys_addr;
1098 #endif /* #if __FreeBSD_version > 1102000 */
1099
1100 QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx"
1101 " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
1102 vm_page, vma->vm_pgoff, unmapped_db,
1103 dev->db_size, len);
1104
1105 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
1106 QL_DPRINT11(ha, "Vma_start not page aligned "
1107 "vm_start = %ld vma_end = %ld\n", vma->vm_start,
1108 vma->vm_end);
1109 return -EINVAL;
1110 }
1111
1112 found = qlnxr_search_mmap(ucontext, vm_page, len);
1113 if (!found) {
1114 QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n",
1115 vma->vm_pgoff);
1116 return -EINVAL;
1117 }
1118
1119 QL_DPRINT12(ha, "Mapping doorbell bar\n");
1120
1121 #if __FreeBSD_version > 1102000
1122
1123 if ((vm_page < unmapped_db) ||
1124 ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) {
1125 QL_DPRINT11(ha, "failed pages are outside of dpi;"
1126 "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n",
1127 vm_page, unmapped_db, ucontext->dpi_size);
1128 return -EINVAL;
1129 }
1130
1131 if (vma->vm_flags & VM_READ) {
1132 QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n");
1133 return -EINVAL;
1134 }
1135
1136 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1137 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
1138 vma->vm_page_prot);
1139
1140 #else
1141
1142 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
1143 dev->db_size))) {
1144 QL_DPRINT12(ha, "Mapping doorbell bar\n");
1145
1146 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1147
1148 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1149 PAGE_SIZE, vma->vm_page_prot);
1150 } else {
1151 QL_DPRINT12(ha, "Mapping chains\n");
1152 rc = io_remap_pfn_range(vma, vma->vm_start,
1153 vma->vm_pgoff, len, vma->vm_page_prot);
1154 }
1155
1156 #endif /* #if __FreeBSD_version > 1102000 */
1157
1158 QL_DPRINT12(ha, "exit [%d]\n", rc);
1159 return rc;
1160 }
1161
1162 struct ib_mr *
qlnxr_get_dma_mr(struct ib_pd * ibpd,int acc)1163 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc)
1164 {
1165 struct qlnxr_mr *mr;
1166 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
1167 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
1168 int rc;
1169 qlnx_host_t *ha;
1170
1171 ha = dev->ha;
1172
1173 QL_DPRINT12(ha, "enter\n");
1174
1175 if (acc & IB_ACCESS_MW_BIND) {
1176 QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n");
1177 }
1178
1179 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1180 if (!mr) {
1181 rc = -ENOMEM;
1182 QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc);
1183 goto err0;
1184 }
1185
1186 mr->type = QLNXR_MR_DMA;
1187
1188 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1189 if (rc) {
1190 QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc);
1191 goto err1;
1192 }
1193
1194 /* index only, 18 bit long, lkey = itid << 8 | key */
1195 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1196 mr->hw_mr.pd = pd->pd_id;
1197 mr->hw_mr.local_read = 1;
1198 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1199 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1200 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1201 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1202 mr->hw_mr.dma_mr = true;
1203
1204 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1205 if (rc) {
1206 QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc);
1207 goto err2;
1208 }
1209
1210 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1211
1212 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1213 mr->hw_mr.remote_atomic) {
1214 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1215 }
1216
1217 QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey);
1218
1219 return &mr->ibmr;
1220
1221 err2:
1222 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1223 err1:
1224 kfree(mr);
1225 err0:
1226 QL_DPRINT12(ha, "exit [%d]\n", rc);
1227
1228 return ERR_PTR(rc);
1229 }
1230
1231 static void
qlnxr_free_pbl(struct qlnxr_dev * dev,struct qlnxr_pbl_info * pbl_info,struct qlnxr_pbl * pbl)1232 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info,
1233 struct qlnxr_pbl *pbl)
1234 {
1235 int i;
1236 qlnx_host_t *ha;
1237
1238 ha = dev->ha;
1239
1240 QL_DPRINT12(ha, "enter\n");
1241
1242 for (i = 0; i < pbl_info->num_pbls; i++) {
1243 if (!pbl[i].va)
1244 continue;
1245 qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa,
1246 pbl_info->pbl_size);
1247 }
1248 kfree(pbl);
1249
1250 QL_DPRINT12(ha, "exit\n");
1251 return;
1252 }
1253
1254 #define MIN_FW_PBL_PAGE_SIZE (4*1024)
1255 #define MAX_FW_PBL_PAGE_SIZE (64*1024)
1256
1257 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
1258 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
1259 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE)
1260
1261 static struct qlnxr_pbl *
qlnxr_alloc_pbl_tbl(struct qlnxr_dev * dev,struct qlnxr_pbl_info * pbl_info,gfp_t flags)1262 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev,
1263 struct qlnxr_pbl_info *pbl_info, gfp_t flags)
1264 {
1265 void *va;
1266 dma_addr_t pa;
1267 dma_addr_t *pbl_main_tbl;
1268 struct qlnxr_pbl *pbl_table;
1269 int i, rc = 0;
1270 qlnx_host_t *ha;
1271
1272 ha = dev->ha;
1273
1274 QL_DPRINT12(ha, "enter\n");
1275
1276 pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags);
1277
1278 if (!pbl_table) {
1279 QL_DPRINT12(ha, "pbl_table = NULL\n");
1280 return NULL;
1281 }
1282
1283 for (i = 0; i < pbl_info->num_pbls; i++) {
1284 va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size);
1285 if (!va) {
1286 QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i);
1287 rc = -ENOMEM;
1288 goto err;
1289 }
1290 memset(va, 0, pbl_info->pbl_size);
1291 pbl_table[i].va = va;
1292 pbl_table[i].pa = pa;
1293 }
1294
1295 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
1296 * the first one with physical pointers to all of the rest
1297 */
1298 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
1299 for (i = 0; i < pbl_info->num_pbls - 1; i++)
1300 pbl_main_tbl[i] = pbl_table[i + 1].pa;
1301
1302 QL_DPRINT12(ha, "exit\n");
1303 return pbl_table;
1304
1305 err:
1306 qlnxr_free_pbl(dev, pbl_info, pbl_table);
1307
1308 QL_DPRINT12(ha, "exit with error\n");
1309 return NULL;
1310 }
1311
1312 static int
qlnxr_prepare_pbl_tbl(struct qlnxr_dev * dev,struct qlnxr_pbl_info * pbl_info,u32 num_pbes,int two_layer_capable)1313 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev,
1314 struct qlnxr_pbl_info *pbl_info,
1315 u32 num_pbes,
1316 int two_layer_capable)
1317 {
1318 u32 pbl_capacity;
1319 u32 pbl_size;
1320 u32 num_pbls;
1321 qlnx_host_t *ha;
1322
1323 ha = dev->ha;
1324
1325 QL_DPRINT12(ha, "enter\n");
1326
1327 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
1328 if (num_pbes > MAX_PBES_TWO_LAYER) {
1329 QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n",
1330 num_pbes);
1331 return -EINVAL;
1332 }
1333
1334 /* calculate required pbl page size */
1335 pbl_size = MIN_FW_PBL_PAGE_SIZE;
1336 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
1337 NUM_PBES_ON_PAGE(pbl_size);
1338
1339 while (pbl_capacity < num_pbes) {
1340 pbl_size *= 2;
1341 pbl_capacity = pbl_size / sizeof(u64);
1342 pbl_capacity = pbl_capacity * pbl_capacity;
1343 }
1344
1345 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
1346 num_pbls++; /* One for the layer0 ( points to the pbls) */
1347 pbl_info->two_layered = true;
1348 } else {
1349 /* One layered PBL */
1350 num_pbls = 1;
1351 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \
1352 roundup_pow_of_two((num_pbes * sizeof(u64))));
1353 pbl_info->two_layered = false;
1354 }
1355
1356 pbl_info->num_pbls = num_pbls;
1357 pbl_info->pbl_size = pbl_size;
1358 pbl_info->num_pbes = num_pbes;
1359
1360 QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n",
1361 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
1362
1363 return 0;
1364 }
1365
1366 static void
qlnxr_populate_pbls(struct qlnxr_dev * dev,struct ib_umem * umem,struct qlnxr_pbl * pbl,struct qlnxr_pbl_info * pbl_info)1367 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem,
1368 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
1369 {
1370 struct regpair *pbe;
1371 struct qlnxr_pbl *pbl_tbl;
1372 struct scatterlist *sg;
1373 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
1374 qlnx_host_t *ha;
1375
1376 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1377 int i;
1378 struct ib_umem_chunk *chunk = NULL;
1379 #else
1380 int entry;
1381 #endif
1382
1383 ha = dev->ha;
1384
1385 QL_DPRINT12(ha, "enter\n");
1386
1387 if (!pbl_info) {
1388 QL_DPRINT11(ha, "PBL_INFO not initialized\n");
1389 return;
1390 }
1391
1392 if (!pbl_info->num_pbes) {
1393 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
1394 return;
1395 }
1396
1397 /* If we have a two layered pbl, the first pbl points to the rest
1398 * of the pbls and the first entry lays on the second pbl in the table
1399 */
1400 if (pbl_info->two_layered)
1401 pbl_tbl = &pbl[1];
1402 else
1403 pbl_tbl = pbl;
1404
1405 pbe = (struct regpair *)pbl_tbl->va;
1406 if (!pbe) {
1407 QL_DPRINT12(ha, "pbe is NULL\n");
1408 return;
1409 }
1410
1411 pbe_cnt = 0;
1412
1413 shift = ilog2(umem->page_size);
1414
1415 #ifndef DEFINE_IB_UMEM_WITH_CHUNK
1416
1417 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
1418 #else
1419 list_for_each_entry(chunk, &umem->chunk_list, list) {
1420 /* get all the dma regions from the chunk. */
1421 for (i = 0; i < chunk->nmap; i++) {
1422 sg = &chunk->page_list[i];
1423 #endif
1424 pages = sg_dma_len(sg) >> shift;
1425 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
1426 /* store the page address in pbe */
1427 pbe->lo =
1428 cpu_to_le32(sg_dma_address(sg) +
1429 (umem->page_size * pg_cnt));
1430 pbe->hi =
1431 cpu_to_le32(upper_32_bits
1432 ((sg_dma_address(sg) +
1433 umem->page_size * pg_cnt)));
1434
1435 QL_DPRINT12(ha,
1436 "Populate pbl table:"
1437 " pbe->addr=0x%x:0x%x "
1438 " pbe_cnt = %d total_num_pbes=%d"
1439 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
1440 total_num_pbes, pbe);
1441
1442 pbe_cnt ++;
1443 total_num_pbes ++;
1444 pbe++;
1445
1446 if (total_num_pbes == pbl_info->num_pbes)
1447 return;
1448
1449 /* if the given pbl is full storing the pbes,
1450 * move to next pbl.
1451 */
1452 if (pbe_cnt ==
1453 (pbl_info->pbl_size / sizeof(u64))) {
1454 pbl_tbl++;
1455 pbe = (struct regpair *)pbl_tbl->va;
1456 pbe_cnt = 0;
1457 }
1458 }
1459 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1460 }
1461 #endif
1462 }
1463 QL_DPRINT12(ha, "exit\n");
1464 return;
1465 }
1466
1467 static void
1468 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info)
1469 {
1470 struct qlnxr_pbl *pbl, *tmp;
1471 qlnx_host_t *ha;
1472
1473 ha = dev->ha;
1474
1475 QL_DPRINT12(ha, "enter\n");
1476
1477 if (info->pbl_table)
1478 list_add_tail(&info->pbl_table->list_entry,
1479 &info->free_pbl_list);
1480
1481 if (!list_empty(&info->inuse_pbl_list))
1482 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
1483
1484 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
1485 list_del(&pbl->list_entry);
1486 qlnxr_free_pbl(dev, &info->pbl_info, pbl);
1487 }
1488 QL_DPRINT12(ha, "exit\n");
1489
1490 return;
1491 }
1492
1493 static int
1494 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info,
1495 size_t page_list_len, bool two_layered)
1496 {
1497 int rc;
1498 struct qlnxr_pbl *tmp;
1499 qlnx_host_t *ha;
1500
1501 ha = dev->ha;
1502
1503 QL_DPRINT12(ha, "enter\n");
1504
1505 INIT_LIST_HEAD(&info->free_pbl_list);
1506 INIT_LIST_HEAD(&info->inuse_pbl_list);
1507
1508 rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info,
1509 page_list_len, two_layered);
1510 if (rc) {
1511 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc);
1512 goto done;
1513 }
1514
1515 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1516
1517 if (!info->pbl_table) {
1518 rc = -ENOMEM;
1519 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n");
1520 goto done;
1521 }
1522
1523 QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa);
1524
1525 /* in usual case we use 2 PBLs, so we add one to free
1526 * list and allocating another one
1527 */
1528 tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1529
1530 if (!tmp) {
1531 QL_DPRINT11(ha, "Extra PBL is not allocated\n");
1532 goto done; /* it's OK if second allocation fails, so rc = 0*/
1533 }
1534
1535 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
1536
1537 QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa);
1538
1539 done:
1540 if (rc)
1541 free_mr_info(dev, info);
1542
1543 QL_DPRINT12(ha, "exit [%d]\n", rc);
1544
1545 return rc;
1546 }
1547
1548 struct ib_mr *
1549 #if __FreeBSD_version >= 1102000
1550 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1551 u64 usr_addr, int acc, struct ib_udata *udata)
1552 #else
1553 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1554 u64 usr_addr, int acc, struct ib_udata *udata, int mr_id)
1555 #endif /* #if __FreeBSD_version >= 1102000 */
1556 {
1557 int rc = -ENOMEM;
1558 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
1559 struct qlnxr_mr *mr;
1560 struct qlnxr_pd *pd;
1561 qlnx_host_t *ha;
1562
1563 ha = dev->ha;
1564
1565 QL_DPRINT12(ha, "enter\n");
1566
1567 pd = get_qlnxr_pd(ibpd);
1568
1569 QL_DPRINT12(ha, "qedr_register user mr pd = %d"
1570 " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
1571 pd->pd_id, start, len, usr_addr, acc);
1572
1573 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
1574 QL_DPRINT11(ha,
1575 "(acc & IB_ACCESS_REMOTE_WRITE &&"
1576 " !(acc & IB_ACCESS_LOCAL_WRITE))\n");
1577 return ERR_PTR(-EINVAL);
1578 }
1579
1580 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1581 if (!mr) {
1582 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
1583 return ERR_PTR(rc);
1584 }
1585
1586 mr->type = QLNXR_MR_USER;
1587
1588 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
1589 if (IS_ERR(mr->umem)) {
1590 rc = -EFAULT;
1591 QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem);
1592 goto err0;
1593 }
1594
1595 rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
1596 if (rc) {
1597 QL_DPRINT11(ha,
1598 "qlnxr_init_mr_info failed [%d]\n", rc);
1599 goto err1;
1600 }
1601
1602 qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
1603 &mr->info.pbl_info);
1604
1605 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1606
1607 if (rc) {
1608 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
1609 goto err1;
1610 }
1611
1612 /* index only, 18 bit long, lkey = itid << 8 | key */
1613 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1614 mr->hw_mr.key = 0;
1615 mr->hw_mr.pd = pd->pd_id;
1616 mr->hw_mr.local_read = 1;
1617 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1618 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1619 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1620 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1621 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
1622 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
1623 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
1624 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
1625 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */
1626
1627 #if __FreeBSD_version >= 1102000
1628 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
1629 #else
1630 mr->hw_mr.fbo = mr->umem->offset;
1631 #endif
1632 mr->hw_mr.length = len;
1633 mr->hw_mr.vaddr = usr_addr;
1634 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
1635 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
1636 mr->hw_mr.dma_mr = false;
1637
1638 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1639 if (rc) {
1640 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
1641 goto err2;
1642 }
1643
1644 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1645 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1646 mr->hw_mr.remote_atomic)
1647 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1648
1649 QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey);
1650
1651 return (&mr->ibmr);
1652
1653 err2:
1654 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1655 err1:
1656 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1657 err0:
1658 kfree(mr);
1659
1660 QL_DPRINT12(ha, "exit [%d]\n", rc);
1661 return (ERR_PTR(rc));
1662 }
1663
1664 int
1665 qlnxr_dereg_mr(struct ib_mr *ib_mr)
1666 {
1667 struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr);
1668 struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device));
1669 int rc = 0;
1670 qlnx_host_t *ha;
1671
1672 ha = dev->ha;
1673
1674 QL_DPRINT12(ha, "enter\n");
1675
1676 if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR))
1677 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1678
1679 /* it could be user registered memory. */
1680 if (mr->umem)
1681 ib_umem_release(mr->umem);
1682
1683 kfree(mr->pages);
1684
1685 kfree(mr);
1686
1687 QL_DPRINT12(ha, "exit\n");
1688 return rc;
1689 }
1690
1691 static int
1692 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev,
1693 struct qlnxr_cq *cq, struct ib_udata *udata)
1694 {
1695 struct qlnxr_create_cq_uresp uresp;
1696 int rc;
1697 qlnx_host_t *ha;
1698
1699 ha = dev->ha;
1700
1701 QL_DPRINT12(ha, "enter\n");
1702
1703 memset(&uresp, 0, sizeof(uresp));
1704
1705 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1706 uresp.icid = cq->icid;
1707
1708 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1709
1710 if (rc) {
1711 QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n",
1712 cq->icid, rc);
1713 }
1714
1715 QL_DPRINT12(ha, "exit [%d]\n", rc);
1716 return rc;
1717 }
1718
1719 static void
1720 consume_cqe(struct qlnxr_cq *cq)
1721 {
1722
1723 if (cq->latest_cqe == cq->toggle_cqe)
1724 cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
1725
1726 cq->latest_cqe = ecore_chain_consume(&cq->pbl);
1727 }
1728
1729 static inline int
1730 qlnxr_align_cq_entries(int entries)
1731 {
1732 u64 size, aligned_size;
1733
1734 /* We allocate an extra entry that we don't report to the FW.
1735 * Why?
1736 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes
1737 * (for performance purposes). Allocating an extra entry and telling
1738 * the FW we have less prevents overwriting the first entry in case of
1739 * a wrap i.e. when the FW writes the last entry and the application
1740 * hasn't read the first one.
1741 */
1742 size = (entries + 1) * QLNXR_CQE_SIZE;
1743
1744 /* We align to PAGE_SIZE.
1745 * Why?
1746 * Since the CQ is going to be mapped and the mapping is anyhow in whole
1747 * kernel pages we benefit from the possibly extra CQEs.
1748 */
1749 aligned_size = ALIGN(size, PAGE_SIZE);
1750
1751 /* note: for CQs created in user space the result of this function
1752 * should match the size mapped in user space
1753 */
1754 return (aligned_size / QLNXR_CQE_SIZE);
1755 }
1756
1757 static inline int
1758 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev,
1759 struct qlnxr_userq *q, u64 buf_addr, size_t buf_len,
1760 int access, int dmasync, int alloc_and_init)
1761 {
1762 int page_cnt;
1763 int rc;
1764 qlnx_host_t *ha;
1765
1766 ha = dev->ha;
1767
1768 QL_DPRINT12(ha, "enter\n");
1769
1770 q->buf_addr = buf_addr;
1771 q->buf_len = buf_len;
1772
1773 QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x"
1774 " dmasync : %x\n", q->buf_addr, q->buf_len,
1775 access, dmasync);
1776
1777 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
1778
1779 if (IS_ERR(q->umem)) {
1780 QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem));
1781 return PTR_ERR(q->umem);
1782 }
1783
1784 page_cnt = ib_umem_page_count(q->umem);
1785 rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt,
1786 0 /* SQ and RQ don't support dual layer pbl.
1787 * CQ may, but this is yet uncoded.
1788 */);
1789 if (rc) {
1790 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc);
1791 goto err;
1792 }
1793
1794 if (alloc_and_init) {
1795 q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
1796
1797 if (!q->pbl_tbl) {
1798 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1799 rc = -ENOMEM;
1800 goto err;
1801 }
1802
1803 qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
1804 } else {
1805 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
1806
1807 if (!q->pbl_tbl) {
1808 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1809 rc = -ENOMEM;
1810 goto err;
1811 }
1812 }
1813
1814 QL_DPRINT12(ha, "exit\n");
1815 return 0;
1816
1817 err:
1818 ib_umem_release(q->umem);
1819 q->umem = NULL;
1820
1821 QL_DPRINT12(ha, "exit [%d]\n", rc);
1822 return rc;
1823 }
1824
1825 #if __FreeBSD_version >= 1102000
1826
1827 struct ib_cq *
1828 qlnxr_create_cq(struct ib_device *ibdev,
1829 const struct ib_cq_init_attr *attr,
1830 struct ib_ucontext *ib_ctx,
1831 struct ib_udata *udata)
1832
1833 #else
1834
1835 #if __FreeBSD_version >= 1100000
1836
1837 struct ib_cq *
1838 qlnxr_create_cq(struct ib_device *ibdev,
1839 struct ib_cq_init_attr *attr,
1840 struct ib_ucontext *ib_ctx,
1841 struct ib_udata *udata)
1842
1843 #else
1844
1845 struct ib_cq *
1846 qlnxr_create_cq(struct ib_device *ibdev,
1847 int entries,
1848 int vector,
1849 struct ib_ucontext *ib_ctx,
1850 struct ib_udata *udata)
1851 #endif /* #if __FreeBSD_version >= 1100000 */
1852
1853 #endif /* #if __FreeBSD_version >= 1102000 */
1854 {
1855 struct qlnxr_ucontext *ctx;
1856 struct ecore_rdma_destroy_cq_out_params destroy_oparams;
1857 struct ecore_rdma_destroy_cq_in_params destroy_iparams;
1858 struct qlnxr_dev *dev;
1859 struct ecore_rdma_create_cq_in_params params;
1860 struct qlnxr_create_cq_ureq ureq;
1861
1862 #if __FreeBSD_version >= 1100000
1863 int vector = attr->comp_vector;
1864 int entries = attr->cqe;
1865 #endif
1866 struct qlnxr_cq *cq;
1867 int chain_entries, rc, page_cnt;
1868 u64 pbl_ptr;
1869 u16 icid;
1870 qlnx_host_t *ha;
1871
1872 dev = get_qlnxr_dev(ibdev);
1873 ha = dev->ha;
1874
1875 QL_DPRINT12(ha, "called from %s. entries = %d, "
1876 "vector = %d\n",
1877 (udata ? "User Lib" : "Kernel"), entries, vector);
1878
1879 memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params));
1880 memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params));
1881 memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params));
1882
1883 if (entries > QLNXR_MAX_CQES) {
1884 QL_DPRINT11(ha,
1885 "the number of entries %d is too high. "
1886 "Must be equal or below %d.\n",
1887 entries, QLNXR_MAX_CQES);
1888 return ERR_PTR(-EINVAL);
1889 }
1890 chain_entries = qlnxr_align_cq_entries(entries);
1891 chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES);
1892
1893 cq = qlnx_zalloc((sizeof(struct qlnxr_cq)));
1894
1895 if (!cq)
1896 return ERR_PTR(-ENOMEM);
1897
1898 if (udata) {
1899 memset(&ureq, 0, sizeof(ureq));
1900
1901 if (ib_copy_from_udata(&ureq, udata,
1902 min(sizeof(ureq), udata->inlen))) {
1903 QL_DPRINT11(ha, "ib_copy_from_udata failed\n");
1904 goto err0;
1905 }
1906
1907 if (!ureq.len) {
1908 QL_DPRINT11(ha, "ureq.len == 0\n");
1909 goto err0;
1910 }
1911
1912 cq->cq_type = QLNXR_CQ_TYPE_USER;
1913
1914 qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len,
1915 IB_ACCESS_LOCAL_WRITE, 1, 1);
1916
1917 pbl_ptr = cq->q.pbl_tbl->pa;
1918 page_cnt = cq->q.pbl_info.num_pbes;
1919 cq->ibcq.cqe = chain_entries;
1920 } else {
1921 cq->cq_type = QLNXR_CQ_TYPE_KERNEL;
1922
1923 rc = ecore_chain_alloc(&dev->ha->cdev,
1924 ECORE_CHAIN_USE_TO_CONSUME,
1925 ECORE_CHAIN_MODE_PBL,
1926 ECORE_CHAIN_CNT_TYPE_U32,
1927 chain_entries,
1928 sizeof(union roce_cqe),
1929 &cq->pbl, NULL);
1930
1931 if (rc)
1932 goto err1;
1933
1934 page_cnt = ecore_chain_get_page_cnt(&cq->pbl);
1935 pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl);
1936 cq->ibcq.cqe = cq->pbl.capacity;
1937 }
1938
1939 params.cq_handle_hi = upper_32_bits((uintptr_t)cq);
1940 params.cq_handle_lo = lower_32_bits((uintptr_t)cq);
1941 params.cnq_id = vector;
1942 params.cq_size = chain_entries - 1;
1943 params.pbl_num_pages = page_cnt;
1944 params.pbl_ptr = pbl_ptr;
1945 params.pbl_two_level = 0;
1946
1947 if (ib_ctx != NULL) {
1948 ctx = get_qlnxr_ucontext(ib_ctx);
1949 params.dpi = ctx->dpi;
1950 } else {
1951 params.dpi = dev->dpi;
1952 }
1953
1954 rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
1955 if (rc)
1956 goto err2;
1957
1958 cq->icid = icid;
1959 cq->sig = QLNXR_CQ_MAGIC_NUMBER;
1960 spin_lock_init(&cq->cq_lock);
1961
1962 if (ib_ctx) {
1963 rc = qlnxr_copy_cq_uresp(dev, cq, udata);
1964 if (rc)
1965 goto err3;
1966 } else {
1967 /* Generate doorbell address.
1968 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT.
1969 * TODO: consider moving to device scope as it is a function of
1970 * the device.
1971 * TODO: add ifdef if plan to support 16 bit.
1972 */
1973 cq->db_addr = dev->db_addr +
1974 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1975 cq->db.data.icid = cq->icid;
1976 cq->db.data.params = DB_AGG_CMD_SET <<
1977 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1978
1979 /* point to the very last element, passing it we will toggle */
1980 cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl);
1981 cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
1982
1983 /* must be different from pbl_toggle */
1984 cq->latest_cqe = NULL;
1985 consume_cqe(cq);
1986 cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
1987 }
1988
1989 QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p,"
1990 " number of entries = 0x%x\n",
1991 cq->icid, cq, params.cq_size);
1992 QL_DPRINT12(ha,"cq_addr = %p\n", cq);
1993 return &cq->ibcq;
1994
1995 err3:
1996 destroy_iparams.icid = cq->icid;
1997 ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams);
1998 err2:
1999 if (udata)
2000 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
2001 else
2002 ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2003 err1:
2004 if (udata)
2005 ib_umem_release(cq->q.umem);
2006 err0:
2007 kfree(cq);
2008
2009 QL_DPRINT12(ha, "exit error\n");
2010
2011 return ERR_PTR(-EINVAL);
2012 }
2013
2014 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
2015 {
2016 int status = 0;
2017 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
2018 qlnx_host_t *ha;
2019
2020 ha = dev->ha;
2021
2022 QL_DPRINT12(ha, "enter/exit\n");
2023
2024 return status;
2025 }
2026
2027 int
2028 qlnxr_destroy_cq(struct ib_cq *ibcq)
2029 {
2030 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
2031 struct ecore_rdma_destroy_cq_out_params oparams;
2032 struct ecore_rdma_destroy_cq_in_params iparams;
2033 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
2034 int rc = 0;
2035 qlnx_host_t *ha;
2036
2037 ha = dev->ha;
2038
2039 QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid);
2040
2041 cq->destroyed = 1;
2042
2043 /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation
2044 * that all completions with notification are dealt with. The rest
2045 * of the completions are not interesting
2046 */
2047
2048 /* GSIs CQs are handled by driver, so they don't exist in the FW */
2049
2050 if (cq->cq_type != QLNXR_CQ_TYPE_GSI) {
2051 iparams.icid = cq->icid;
2052
2053 rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
2054
2055 if (rc) {
2056 QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n",
2057 cq->icid);
2058 return rc;
2059 }
2060
2061 QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid);
2062 ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2063 }
2064
2065 if (ibcq->uobject && ibcq->uobject->context) {
2066 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
2067 ib_umem_release(cq->q.umem);
2068 }
2069
2070 cq->sig = ~cq->sig;
2071
2072 kfree(cq);
2073
2074 QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid);
2075
2076 return rc;
2077 }
2078
2079 static int
2080 qlnxr_check_qp_attrs(struct ib_pd *ibpd,
2081 struct qlnxr_dev *dev,
2082 struct ib_qp_init_attr *attrs,
2083 struct ib_udata *udata)
2084 {
2085 struct ecore_rdma_device *qattr;
2086 qlnx_host_t *ha;
2087
2088 qattr = ecore_rdma_query_device(dev->rdma_ctx);
2089 ha = dev->ha;
2090
2091 QL_DPRINT12(ha, "enter\n");
2092
2093 QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type);
2094 QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type);
2095 QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags);
2096
2097 #if __FreeBSD_version < 1102000
2098 QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type);
2099 #endif
2100
2101 QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num);
2102 QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr);
2103 QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr);
2104 QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge);
2105 QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge);
2106 QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n",
2107 attrs->cap.max_inline_data);
2108
2109 #if __FreeBSD_version < 1102000
2110 QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n",
2111 attrs->cap.qpg_tss_mask_sz);
2112 #endif
2113
2114 QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id);
2115 QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id);
2116 QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver);
2117 QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver);
2118 QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid);
2119 QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n",
2120 (void *)qattr->sys_image_guid);
2121 QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq);
2122 QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge);
2123 QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge);
2124 QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline);
2125 QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe);
2126 QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe);
2127 QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n",
2128 qattr->max_qp_resp_rd_atomic_resc);
2129 QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n",
2130 qattr->max_qp_req_rd_atomic_resc);
2131 QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n",
2132 qattr->max_dev_resp_rd_atomic_resc);
2133 QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq);
2134 QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp);
2135 QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq);
2136 QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr);
2137 QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size);
2138 QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe);
2139 QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw);
2140 QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr);
2141 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n",
2142 qattr->max_mr_mw_fmr_pbl);
2143 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n",
2144 (void *)qattr->max_mr_mw_fmr_size);
2145 QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd);
2146 QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah);
2147 QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey);
2148 QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr);
2149 QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n",
2150 qattr->max_stats_queues);
2151 //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps);
2152 QL_DPRINT12(ha, "qattr->page_size_caps = %p\n",
2153 (void *)qattr->page_size_caps);
2154 QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay);
2155 QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey);
2156 QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n",
2157 qattr->bad_pkey_counter);
2158
2159 if ((attrs->qp_type == IB_QPT_GSI) && udata) {
2160 QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n");
2161 return -EINVAL;
2162 }
2163
2164 if (udata && !(ibpd->uobject && ibpd->uobject->context)) {
2165 QL_DPRINT12(ha, "called from user without context\n");
2166 return -EINVAL;
2167 }
2168
2169 /* QP0... attrs->qp_type == IB_QPT_GSI */
2170 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
2171 QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n",
2172 attrs->qp_type);
2173 return -EINVAL;
2174 }
2175 if (attrs->qp_type == IB_QPT_GSI && attrs->srq) {
2176 QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n");
2177 return -EINVAL;
2178 }
2179 /* Skip the check for QP1 to support CM size of 128 */
2180 if (attrs->cap.max_send_wr > qattr->max_wqe) {
2181 QL_DPRINT12(ha, "cannot create a SQ with %d elements "
2182 " (max_send_wr=0x%x)\n",
2183 attrs->cap.max_send_wr, qattr->max_wqe);
2184 return -EINVAL;
2185 }
2186 if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) {
2187 QL_DPRINT12(ha, "cannot create a RQ with %d elements"
2188 " (max_recv_wr=0x%x)\n",
2189 attrs->cap.max_recv_wr, qattr->max_wqe);
2190 return -EINVAL;
2191 }
2192 if (attrs->cap.max_inline_data > qattr->max_inline) {
2193 QL_DPRINT12(ha,
2194 "unsupported inline data size=0x%x "
2195 "requested (max_inline=0x%x)\n",
2196 attrs->cap.max_inline_data, qattr->max_inline);
2197 return -EINVAL;
2198 }
2199 if (attrs->cap.max_send_sge > qattr->max_sge) {
2200 QL_DPRINT12(ha,
2201 "unsupported send_sge=0x%x "
2202 "requested (max_send_sge=0x%x)\n",
2203 attrs->cap.max_send_sge, qattr->max_sge);
2204 return -EINVAL;
2205 }
2206 if (attrs->cap.max_recv_sge > qattr->max_sge) {
2207 QL_DPRINT12(ha,
2208 "unsupported recv_sge=0x%x requested "
2209 " (max_recv_sge=0x%x)\n",
2210 attrs->cap.max_recv_sge, qattr->max_sge);
2211 return -EINVAL;
2212 }
2213 /* unprivileged user space cannot create special QP */
2214 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
2215 QL_DPRINT12(ha,
2216 "userspace can't create special QPs of type=0x%x\n",
2217 attrs->qp_type);
2218 return -EINVAL;
2219 }
2220 /* allow creating only one GSI type of QP */
2221 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
2222 QL_DPRINT12(ha,
2223 "create qp: GSI special QPs already created.\n");
2224 return -EINVAL;
2225 }
2226
2227 /* verify consumer QPs are not trying to use GSI QP's CQ */
2228 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
2229 struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq);
2230 struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq);
2231
2232 if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) ||
2233 (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) {
2234 QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n");
2235 return -EINVAL;
2236 }
2237 }
2238 QL_DPRINT12(ha, "exit\n");
2239 return 0;
2240 }
2241
2242 static int
2243 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
2244 struct qlnxr_srq *srq,
2245 struct ib_udata *udata)
2246 {
2247 struct qlnxr_create_srq_uresp uresp;
2248 qlnx_host_t *ha;
2249 int rc;
2250
2251 ha = dev->ha;
2252
2253 QL_DPRINT12(ha, "enter\n");
2254
2255 memset(&uresp, 0, sizeof(uresp));
2256
2257 uresp.srq_id = srq->srq_id;
2258
2259 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2260
2261 QL_DPRINT12(ha, "exit [%d]\n", rc);
2262 return rc;
2263 }
2264
2265 static void
2266 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev,
2267 struct qlnxr_create_qp_uresp *uresp,
2268 struct qlnxr_qp *qp)
2269 {
2270 qlnx_host_t *ha;
2271
2272 ha = dev->ha;
2273
2274 /* Return if QP is associated with SRQ instead of RQ */
2275 QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq);
2276
2277 if (qp->srq)
2278 return;
2279
2280 /* iWARP requires two doorbells per RQ. */
2281 if (QLNX_IS_IWARP(dev)) {
2282 uresp->rq_db_offset =
2283 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2284 uresp->rq_db2_offset =
2285 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2286
2287 QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x "
2288 "uresp->rq_db2_offset = 0x%x\n",
2289 uresp->rq_db_offset, uresp->rq_db2_offset);
2290 } else {
2291 uresp->rq_db_offset =
2292 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2293 }
2294 uresp->rq_icid = qp->icid;
2295
2296 QL_DPRINT12(ha, "exit\n");
2297 return;
2298 }
2299
2300 static void
2301 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev,
2302 struct qlnxr_create_qp_uresp *uresp,
2303 struct qlnxr_qp *qp)
2304 {
2305 qlnx_host_t *ha;
2306
2307 ha = dev->ha;
2308
2309 QL_DPRINT12(ha, "enter\n");
2310
2311 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2312
2313 /* iWARP uses the same cid for rq and sq*/
2314 if (QLNX_IS_IWARP(dev)) {
2315 uresp->sq_icid = qp->icid;
2316 QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid);
2317 } else
2318 uresp->sq_icid = qp->icid + 1;
2319
2320 QL_DPRINT12(ha, "exit\n");
2321 return;
2322 }
2323
2324 static int
2325 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev,
2326 struct qlnxr_qp *qp,
2327 struct ib_udata *udata)
2328 {
2329 int rc;
2330 struct qlnxr_create_qp_uresp uresp;
2331 qlnx_host_t *ha;
2332
2333 ha = dev->ha;
2334
2335 QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid);
2336
2337 memset(&uresp, 0, sizeof(uresp));
2338 qlnxr_copy_sq_uresp(dev, &uresp, qp);
2339 qlnxr_copy_rq_uresp(dev, &uresp, qp);
2340
2341 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
2342 uresp.qp_id = qp->qp_id;
2343
2344 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2345
2346 QL_DPRINT12(ha, "exit [%d]\n", rc);
2347 return rc;
2348 }
2349
2350 static void
2351 qlnxr_set_common_qp_params(struct qlnxr_dev *dev,
2352 struct qlnxr_qp *qp,
2353 struct qlnxr_pd *pd,
2354 struct ib_qp_init_attr *attrs)
2355 {
2356 qlnx_host_t *ha;
2357
2358 ha = dev->ha;
2359
2360 QL_DPRINT12(ha, "enter\n");
2361
2362 spin_lock_init(&qp->q_lock);
2363
2364 atomic_set(&qp->refcnt, 1);
2365 qp->pd = pd;
2366 qp->sig = QLNXR_QP_MAGIC_NUMBER;
2367 qp->qp_type = attrs->qp_type;
2368 qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2369 qp->sq.max_sges = attrs->cap.max_send_sge;
2370 qp->state = ECORE_ROCE_QP_STATE_RESET;
2371 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
2372 qp->sq_cq = get_qlnxr_cq(attrs->send_cq);
2373 qp->rq_cq = get_qlnxr_cq(attrs->recv_cq);
2374 qp->dev = dev;
2375
2376 if (!attrs->srq) {
2377 /* QP is associated with RQ instead of SRQ */
2378 qp->rq.max_sges = attrs->cap.max_recv_sge;
2379 QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
2380 qp->rq.max_sges, qp->rq_cq->icid);
2381 } else {
2382 qp->srq = get_qlnxr_srq(attrs->srq);
2383 }
2384
2385 QL_DPRINT12(ha,
2386 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d,"
2387 " state = %d, signaled = %d, use_srq=%d\n",
2388 pd->pd_id, qp->qp_type, qp->max_inline_data,
2389 qp->state, qp->signaled, ((attrs->srq) ? 1 : 0));
2390 QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
2391 qp->sq.max_sges, qp->sq_cq->icid);
2392 return;
2393 }
2394
2395 static int
2396 qlnxr_check_srq_params(struct ib_pd *ibpd,
2397 struct qlnxr_dev *dev,
2398 struct ib_srq_init_attr *attrs)
2399 {
2400 struct ecore_rdma_device *qattr;
2401 qlnx_host_t *ha;
2402
2403 ha = dev->ha;
2404 qattr = ecore_rdma_query_device(dev->rdma_ctx);
2405
2406 QL_DPRINT12(ha, "enter\n");
2407
2408 if (attrs->attr.max_wr > qattr->max_srq_wqe) {
2409 QL_DPRINT12(ha, "unsupported srq_wr=0x%x"
2410 " requested (max_srq_wr=0x%x)\n",
2411 attrs->attr.max_wr, qattr->max_srq_wr);
2412 return -EINVAL;
2413 }
2414
2415 if (attrs->attr.max_sge > qattr->max_sge) {
2416 QL_DPRINT12(ha,
2417 "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
2418 attrs->attr.max_sge, qattr->max_sge);
2419 return -EINVAL;
2420 }
2421
2422 if (attrs->attr.srq_limit > attrs->attr.max_wr) {
2423 QL_DPRINT12(ha,
2424 "unsupported srq_limit=0x%x requested"
2425 " (max_srq_limit=0x%x)\n",
2426 attrs->attr.srq_limit, attrs->attr.srq_limit);
2427 return -EINVAL;
2428 }
2429
2430 QL_DPRINT12(ha, "exit\n");
2431 return 0;
2432 }
2433
2434 static void
2435 qlnxr_free_srq_user_params(struct qlnxr_srq *srq)
2436 {
2437 struct qlnxr_dev *dev = srq->dev;
2438 qlnx_host_t *ha;
2439
2440 ha = dev->ha;
2441
2442 QL_DPRINT12(ha, "enter\n");
2443
2444 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2445 ib_umem_release(srq->usrq.umem);
2446 ib_umem_release(srq->prod_umem);
2447
2448 QL_DPRINT12(ha, "exit\n");
2449 return;
2450 }
2451
2452 static void
2453 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq)
2454 {
2455 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq;
2456 struct qlnxr_dev *dev = srq->dev;
2457 qlnx_host_t *ha;
2458
2459 ha = dev->ha;
2460
2461 QL_DPRINT12(ha, "enter\n");
2462
2463 ecore_chain_free(dev->cdev, &hw_srq->pbl);
2464
2465 qlnx_dma_free_coherent(&dev->cdev,
2466 hw_srq->virt_prod_pair_addr,
2467 hw_srq->phy_prod_pair_addr,
2468 sizeof(struct rdma_srq_producers));
2469
2470 QL_DPRINT12(ha, "exit\n");
2471
2472 return;
2473 }
2474
2475 static int
2476 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
2477 struct qlnxr_srq *srq,
2478 struct qlnxr_create_srq_ureq *ureq,
2479 int access, int dmasync)
2480 {
2481 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2482 struct ib_umem_chunk *chunk;
2483 #endif
2484 struct scatterlist *sg;
2485 int rc;
2486 struct qlnxr_dev *dev = srq->dev;
2487 qlnx_host_t *ha;
2488
2489 ha = dev->ha;
2490
2491 QL_DPRINT12(ha, "enter\n");
2492
2493 rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,
2494 ureq->srq_len, access, dmasync, 1);
2495 if (rc)
2496 return rc;
2497
2498 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,
2499 sizeof(struct rdma_srq_producers),
2500 access, dmasync);
2501 if (IS_ERR(srq->prod_umem)) {
2502 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2503 ib_umem_release(srq->usrq.umem);
2504
2505 QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n",
2506 PTR_ERR(srq->prod_umem));
2507
2508 return PTR_ERR(srq->prod_umem);
2509 }
2510
2511 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2512 chunk = container_of((&srq->prod_umem->chunk_list)->next,
2513 typeof(*chunk), list);
2514 sg = &chunk->page_list[0];
2515 #else
2516 sg = srq->prod_umem->sg_head.sgl;
2517 #endif
2518 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
2519
2520 QL_DPRINT12(ha, "exit\n");
2521 return 0;
2522 }
2523
2524 static int
2525 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
2526 struct qlnxr_dev *dev,
2527 struct ib_srq_init_attr *init_attr)
2528 {
2529 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq;
2530 dma_addr_t phy_prod_pair_addr;
2531 u32 num_elems, max_wr;
2532 void *va;
2533 int rc;
2534 qlnx_host_t *ha;
2535
2536 ha = dev->ha;
2537
2538 QL_DPRINT12(ha, "enter\n");
2539
2540 va = qlnx_dma_alloc_coherent(&dev->cdev,
2541 &phy_prod_pair_addr,
2542 sizeof(struct rdma_srq_producers));
2543 if (!va) {
2544 QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n");
2545 return -ENOMEM;
2546 }
2547
2548 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
2549 hw_srq->virt_prod_pair_addr = va;
2550
2551 max_wr = init_attr->attr.max_wr;
2552
2553 num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE;
2554
2555 rc = ecore_chain_alloc(dev->cdev,
2556 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2557 ECORE_CHAIN_MODE_PBL,
2558 ECORE_CHAIN_CNT_TYPE_U32,
2559 num_elems,
2560 ECORE_RDMA_SRQ_WQE_ELEM_SIZE,
2561 &hw_srq->pbl, NULL);
2562
2563 if (rc) {
2564 QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc);
2565 goto err0;
2566 }
2567
2568 hw_srq->max_wr = max_wr;
2569 hw_srq->num_elems = num_elems;
2570 hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ;
2571
2572 QL_DPRINT12(ha, "exit\n");
2573 return 0;
2574
2575 err0:
2576 qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr,
2577 sizeof(struct rdma_srq_producers));
2578
2579 QL_DPRINT12(ha, "exit [%d]\n", rc);
2580 return rc;
2581 }
2582
2583 static inline void
2584 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev,
2585 struct qlnxr_pd *pd,
2586 struct qlnxr_qp *qp,
2587 struct ib_qp_init_attr *attrs,
2588 bool fmr_and_reserved_lkey,
2589 struct ecore_rdma_create_qp_in_params *params)
2590 {
2591 qlnx_host_t *ha;
2592
2593 ha = dev->ha;
2594
2595 QL_DPRINT12(ha, "enter\n");
2596
2597 /* QP handle to be written in an async event */
2598 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
2599 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
2600
2601 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
2602 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
2603 params->pd = pd->pd_id;
2604 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
2605 params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
2606 params->stats_queue = 0;
2607
2608 params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
2609
2610 if (qp->srq) {
2611 /* QP is associated with SRQ instead of RQ */
2612 params->srq_id = qp->srq->srq_id;
2613 params->use_srq = true;
2614 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
2615 params->srq_id, params->use_srq);
2616 return;
2617 }
2618
2619 params->srq_id = 0;
2620 params->use_srq = false;
2621
2622 QL_DPRINT12(ha, "exit\n");
2623 return;
2624 }
2625
2626 static inline void
2627 qlnxr_qp_user_print( struct qlnxr_dev *dev,
2628 struct qlnxr_qp *qp)
2629 {
2630 QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, "
2631 "rq_addr=0x%llx, rq_len=%zd\n",
2632 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
2633 qp->urq.buf_len);
2634 return;
2635 }
2636
2637 static int
2638 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id)
2639 {
2640 u32 newid;
2641 int rc;
2642 qlnx_host_t *ha;
2643
2644 ha = dev->ha;
2645
2646 QL_DPRINT12(ha, "enter\n");
2647
2648 if (!QLNX_IS_IWARP(dev))
2649 return 0;
2650
2651 do {
2652 if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) {
2653 QL_DPRINT11(ha, "idr_pre_get failed\n");
2654 return -ENOMEM;
2655 }
2656
2657 mtx_lock(&dev->idr_lock);
2658
2659 rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid);
2660
2661 mtx_unlock(&dev->idr_lock);
2662
2663 } while (rc == -EAGAIN);
2664
2665 QL_DPRINT12(ha, "exit [%d]\n", rc);
2666
2667 return rc;
2668 }
2669
2670 static void
2671 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id)
2672 {
2673 qlnx_host_t *ha;
2674
2675 ha = dev->ha;
2676
2677 QL_DPRINT12(ha, "enter\n");
2678
2679 if (!QLNX_IS_IWARP(dev))
2680 return;
2681
2682 mtx_lock(&dev->idr_lock);
2683 idr_remove(&dev->qpidr, id);
2684 mtx_unlock(&dev->idr_lock);
2685
2686 QL_DPRINT12(ha, "exit \n");
2687
2688 return;
2689 }
2690
2691 static inline void
2692 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev,
2693 struct qlnxr_qp *qp,
2694 struct ecore_rdma_create_qp_out_params *out_params)
2695 {
2696 qlnx_host_t *ha;
2697
2698 ha = dev->ha;
2699
2700 QL_DPRINT12(ha, "enter\n");
2701
2702 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
2703 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
2704
2705 qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
2706 &qp->usq.pbl_info);
2707
2708 if (qp->srq) {
2709 QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq);
2710 return;
2711 }
2712
2713 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
2714 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
2715
2716 qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
2717 &qp->urq.pbl_info);
2718
2719 QL_DPRINT12(ha, "exit\n");
2720 return;
2721 }
2722
2723 static int
2724 qlnxr_create_user_qp(struct qlnxr_dev *dev,
2725 struct qlnxr_qp *qp,
2726 struct ib_pd *ibpd,
2727 struct ib_udata *udata,
2728 struct ib_qp_init_attr *attrs)
2729 {
2730 struct ecore_rdma_destroy_qp_out_params d_out_params;
2731 struct ecore_rdma_create_qp_in_params in_params;
2732 struct ecore_rdma_create_qp_out_params out_params;
2733 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
2734 struct ib_ucontext *ib_ctx = NULL;
2735 struct qlnxr_ucontext *ctx = NULL;
2736 struct qlnxr_create_qp_ureq ureq;
2737 int alloc_and_init = QLNX_IS_ROCE(dev);
2738 int rc = -EINVAL;
2739 qlnx_host_t *ha;
2740
2741 ha = dev->ha;
2742
2743 QL_DPRINT12(ha, "enter\n");
2744
2745 ib_ctx = ibpd->uobject->context;
2746 ctx = get_qlnxr_ucontext(ib_ctx);
2747
2748 memset(&ureq, 0, sizeof(ureq));
2749 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
2750
2751 if (rc) {
2752 QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc);
2753 return rc;
2754 }
2755
2756 /* SQ - read access only (0), dma sync not required (0) */
2757 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
2758 ureq.sq_len, 0, 0,
2759 alloc_and_init);
2760 if (rc) {
2761 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2762 return rc;
2763 }
2764
2765 if (!qp->srq) {
2766 /* RQ - read access only (0), dma sync not required (0) */
2767 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
2768 ureq.rq_len, 0, 0,
2769 alloc_and_init);
2770
2771 if (rc) {
2772 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2773 return rc;
2774 }
2775 }
2776
2777 memset(&in_params, 0, sizeof(in_params));
2778 qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
2779 in_params.qp_handle_lo = ureq.qp_handle_lo;
2780 in_params.qp_handle_hi = ureq.qp_handle_hi;
2781 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
2782 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
2783
2784 if (!qp->srq) {
2785 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
2786 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
2787 }
2788
2789 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params);
2790
2791 if (!qp->ecore_qp) {
2792 rc = -ENOMEM;
2793 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
2794 goto err1;
2795 }
2796
2797 if (QLNX_IS_IWARP(dev))
2798 qlnxr_iwarp_populate_user_qp(dev, qp, &out_params);
2799
2800 qp->qp_id = out_params.qp_id;
2801 qp->icid = out_params.icid;
2802
2803 rc = qlnxr_copy_qp_uresp(dev, qp, udata);
2804
2805 if (rc) {
2806 QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n");
2807 goto err;
2808 }
2809
2810 qlnxr_qp_user_print(dev, qp);
2811
2812 QL_DPRINT12(ha, "exit\n");
2813 return 0;
2814 err:
2815 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
2816
2817 if (rc)
2818 QL_DPRINT12(ha, "fatal fault\n");
2819
2820 err1:
2821 qlnxr_cleanup_user(dev, qp);
2822
2823 QL_DPRINT12(ha, "exit[%d]\n", rc);
2824 return rc;
2825 }
2826
2827 static void
2828 qlnxr_set_roce_db_info(struct qlnxr_dev *dev,
2829 struct qlnxr_qp *qp)
2830 {
2831 qlnx_host_t *ha;
2832
2833 ha = dev->ha;
2834
2835 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2836
2837 qp->sq.db = dev->db_addr +
2838 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2839 qp->sq.db_data.data.icid = qp->icid + 1;
2840
2841 if (!qp->srq) {
2842 qp->rq.db = dev->db_addr +
2843 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2844 qp->rq.db_data.data.icid = qp->icid;
2845 }
2846
2847 QL_DPRINT12(ha, "exit\n");
2848 return;
2849 }
2850
2851 static void
2852 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev,
2853 struct qlnxr_qp *qp)
2854
2855 {
2856 qlnx_host_t *ha;
2857
2858 ha = dev->ha;
2859
2860 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2861
2862 qp->sq.db = dev->db_addr +
2863 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2864 qp->sq.db_data.data.icid = qp->icid;
2865
2866 if (!qp->srq) {
2867 qp->rq.db = dev->db_addr +
2868 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2869 qp->rq.db_data.data.icid = qp->icid;
2870
2871 qp->rq.iwarp_db2 = dev->db_addr +
2872 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2873 qp->rq.iwarp_db2_data.data.icid = qp->icid;
2874 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2875 }
2876
2877 QL_DPRINT12(ha,
2878 "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n"
2879 "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n"
2880 "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x"
2881 " qp->rq.iwarp_db2.data.prod_val =0x%x\n",
2882 qp->sq.db, qp->sq.db_data.data.icid,
2883 qp->rq.db, qp->rq.db_data.data.icid,
2884 qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid,
2885 qp->rq.iwarp_db2_data.data.value);
2886
2887 QL_DPRINT12(ha, "exit\n");
2888 return;
2889 }
2890
2891 static int
2892 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev,
2893 struct qlnxr_qp *qp,
2894 struct ecore_rdma_create_qp_in_params *in_params,
2895 u32 n_sq_elems,
2896 u32 n_rq_elems)
2897 {
2898 struct ecore_rdma_create_qp_out_params out_params;
2899 int rc;
2900 qlnx_host_t *ha;
2901
2902 ha = dev->ha;
2903
2904 QL_DPRINT12(ha, "enter\n");
2905
2906 rc = ecore_chain_alloc(
2907 dev->cdev,
2908 ECORE_CHAIN_USE_TO_PRODUCE,
2909 ECORE_CHAIN_MODE_PBL,
2910 ECORE_CHAIN_CNT_TYPE_U32,
2911 n_sq_elems,
2912 QLNXR_SQE_ELEMENT_SIZE,
2913 &qp->sq.pbl,
2914 NULL);
2915
2916 if (rc) {
2917 QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc);
2918 return rc;
2919 }
2920
2921 in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl);
2922 in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl);
2923
2924 if (!qp->srq) {
2925 rc = ecore_chain_alloc(
2926 dev->cdev,
2927 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2928 ECORE_CHAIN_MODE_PBL,
2929 ECORE_CHAIN_CNT_TYPE_U32,
2930 n_rq_elems,
2931 QLNXR_RQE_ELEMENT_SIZE,
2932 &qp->rq.pbl,
2933 NULL);
2934
2935 if (rc) {
2936 QL_DPRINT11(ha,
2937 "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc);
2938 return rc;
2939 }
2940
2941 in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl);
2942 in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl);
2943 }
2944
2945 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
2946
2947 if (!qp->ecore_qp) {
2948 QL_DPRINT11(ha, "qp->ecore_qp == NULL\n");
2949 return -EINVAL;
2950 }
2951
2952 qp->qp_id = out_params.qp_id;
2953 qp->icid = out_params.icid;
2954
2955 qlnxr_set_roce_db_info(dev, qp);
2956
2957 QL_DPRINT12(ha, "exit\n");
2958 return 0;
2959 }
2960
2961 static int
2962 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev,
2963 struct qlnxr_qp *qp,
2964 struct ecore_rdma_create_qp_in_params *in_params,
2965 u32 n_sq_elems,
2966 u32 n_rq_elems)
2967 {
2968 struct ecore_rdma_destroy_qp_out_params d_out_params;
2969 struct ecore_rdma_create_qp_out_params out_params;
2970 struct ecore_chain_ext_pbl ext_pbl;
2971 int rc;
2972 qlnx_host_t *ha;
2973
2974 ha = dev->ha;
2975
2976 QL_DPRINT12(ha, "enter\n");
2977
2978 in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems,
2979 QLNXR_SQE_ELEMENT_SIZE,
2980 ECORE_CHAIN_MODE_PBL);
2981 in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems,
2982 QLNXR_RQE_ELEMENT_SIZE,
2983 ECORE_CHAIN_MODE_PBL);
2984
2985 QL_DPRINT12(ha, "n_sq_elems = 0x%x"
2986 " n_rq_elems = 0x%x in_params\n"
2987 "\t\t\tqp_handle_lo\t\t= 0x%08x\n"
2988 "\t\t\tqp_handle_hi\t\t= 0x%08x\n"
2989 "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n"
2990 "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n"
2991 "\t\t\tuse_srq\t\t\t= 0x%x\n"
2992 "\t\t\tsignal_all\t\t= 0x%x\n"
2993 "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n"
2994 "\t\t\tpd\t\t\t= 0x%x\n"
2995 "\t\t\tdpi\t\t\t= 0x%x\n"
2996 "\t\t\tsq_cq_id\t\t\t= 0x%x\n"
2997 "\t\t\tsq_num_pages\t\t= 0x%x\n"
2998 "\t\t\tsq_pbl_ptr\t\t= %p\n"
2999 "\t\t\tmax_sq_sges\t\t= 0x%x\n"
3000 "\t\t\trq_cq_id\t\t\t= 0x%x\n"
3001 "\t\t\trq_num_pages\t\t= 0x%x\n"
3002 "\t\t\trq_pbl_ptr\t\t= %p\n"
3003 "\t\t\tsrq_id\t\t\t= 0x%x\n"
3004 "\t\t\tstats_queue\t\t= 0x%x\n",
3005 n_sq_elems, n_rq_elems,
3006 in_params->qp_handle_lo,
3007 in_params->qp_handle_hi,
3008 in_params->qp_handle_async_lo,
3009 in_params->qp_handle_async_hi,
3010 in_params->use_srq,
3011 in_params->signal_all,
3012 in_params->fmr_and_reserved_lkey,
3013 in_params->pd,
3014 in_params->dpi,
3015 in_params->sq_cq_id,
3016 in_params->sq_num_pages,
3017 (void *)in_params->sq_pbl_ptr,
3018 in_params->max_sq_sges,
3019 in_params->rq_cq_id,
3020 in_params->rq_num_pages,
3021 (void *)in_params->rq_pbl_ptr,
3022 in_params->srq_id,
3023 in_params->stats_queue );
3024
3025 memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params));
3026 memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl));
3027
3028 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
3029
3030 if (!qp->ecore_qp) {
3031 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
3032 return -EINVAL;
3033 }
3034
3035 /* Now we allocate the chain */
3036 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
3037 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
3038
3039 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3040 "ext_pbl.p_pbl_phys = %p\n",
3041 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3042
3043 rc = ecore_chain_alloc(
3044 dev->cdev,
3045 ECORE_CHAIN_USE_TO_PRODUCE,
3046 ECORE_CHAIN_MODE_PBL,
3047 ECORE_CHAIN_CNT_TYPE_U32,
3048 n_sq_elems,
3049 QLNXR_SQE_ELEMENT_SIZE,
3050 &qp->sq.pbl,
3051 &ext_pbl);
3052
3053 if (rc) {
3054 QL_DPRINT11(ha,
3055 "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc);
3056 goto err;
3057 }
3058
3059 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
3060 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
3061
3062 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3063 "ext_pbl.p_pbl_phys = %p\n",
3064 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3065
3066 if (!qp->srq) {
3067 rc = ecore_chain_alloc(
3068 dev->cdev,
3069 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
3070 ECORE_CHAIN_MODE_PBL,
3071 ECORE_CHAIN_CNT_TYPE_U32,
3072 n_rq_elems,
3073 QLNXR_RQE_ELEMENT_SIZE,
3074 &qp->rq.pbl,
3075 &ext_pbl);
3076
3077 if (rc) {
3078 QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl"
3079 " failed rc = %d\n", rc);
3080 goto err;
3081 }
3082 }
3083
3084 QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n",
3085 out_params.qp_id, out_params.icid);
3086
3087 qp->qp_id = out_params.qp_id;
3088 qp->icid = out_params.icid;
3089
3090 qlnxr_set_iwarp_db_info(dev, qp);
3091
3092 QL_DPRINT12(ha, "exit\n");
3093 return 0;
3094
3095 err:
3096 ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
3097
3098 QL_DPRINT12(ha, "exit rc = %d\n", rc);
3099 return rc;
3100 }
3101
3102 static int
3103 qlnxr_create_kernel_qp(struct qlnxr_dev *dev,
3104 struct qlnxr_qp *qp,
3105 struct ib_pd *ibpd,
3106 struct ib_qp_init_attr *attrs)
3107 {
3108 struct ecore_rdma_create_qp_in_params in_params;
3109 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3110 int rc = -EINVAL;
3111 u32 n_rq_elems;
3112 u32 n_sq_elems;
3113 u32 n_sq_entries;
3114 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3115 qlnx_host_t *ha;
3116
3117 ha = dev->ha;
3118
3119 QL_DPRINT12(ha, "enter\n");
3120
3121 memset(&in_params, 0, sizeof(in_params));
3122
3123 /* A single work request may take up to MAX_SQ_WQE_SIZE elements in
3124 * the ring. The ring should allow at least a single WR, even if the
3125 * user requested none, due to allocation issues.
3126 * We should add an extra WR since the prod and cons indices of
3127 * wqe_wr_id are managed in such a way that the WQ is considered full
3128 * when (prod+1)%max_wr==cons. We currently don't do that because we
3129 * double the number of entries due an iSER issue that pushes far more
3130 * WRs than indicated. If we decline its ib_post_send() then we get
3131 * error prints in the dmesg we'd like to avoid.
3132 */
3133 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
3134 qattr->max_wqe);
3135
3136 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
3137 GFP_KERNEL);
3138 if (!qp->wqe_wr_id) {
3139 QL_DPRINT11(ha, "failed SQ shadow memory allocation\n");
3140 return -ENOMEM;
3141 }
3142
3143 /* QP handle to be written in CQE */
3144 in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp);
3145 in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp);
3146
3147 /* A single work request may take up to MAX_RQ_WQE_SIZE elements in
3148 * the ring. There ring should allow at least a single WR, even if the
3149 * user requested none, due to allocation issues.
3150 */
3151 qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1);
3152
3153 /* Allocate driver internal RQ array */
3154 if (!qp->srq) {
3155 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
3156 GFP_KERNEL);
3157 if (!qp->rqe_wr_id) {
3158 QL_DPRINT11(ha, "failed RQ shadow memory allocation\n");
3159 kfree(qp->wqe_wr_id);
3160 return -ENOMEM;
3161 }
3162 }
3163
3164 //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
3165
3166 in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
3167 in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
3168
3169 in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
3170 in_params.fmr_and_reserved_lkey = true;
3171 in_params.pd = pd->pd_id;
3172 in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
3173 in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
3174 in_params.stats_queue = 0;
3175
3176 in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
3177
3178 if (qp->srq) {
3179 /* QP is associated with SRQ instead of RQ */
3180 in_params.srq_id = qp->srq->srq_id;
3181 in_params.use_srq = true;
3182 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
3183 in_params.srq_id, in_params.use_srq);
3184 } else {
3185 in_params.srq_id = 0;
3186 in_params.use_srq = false;
3187 }
3188
3189 n_sq_entries = attrs->cap.max_send_wr;
3190 n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe);
3191 n_sq_entries = max_t(u32, n_sq_entries, 1);
3192 n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
3193
3194 n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
3195
3196 if (QLNX_IS_ROCE(dev)) {
3197 rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params,
3198 n_sq_elems, n_rq_elems);
3199 } else {
3200 rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params,
3201 n_sq_elems, n_rq_elems);
3202 }
3203
3204 if (rc)
3205 qlnxr_cleanup_kernel(dev, qp);
3206
3207 QL_DPRINT12(ha, "exit [%d]\n", rc);
3208 return rc;
3209 }
3210
3211 struct ib_qp *
3212 qlnxr_create_qp(struct ib_pd *ibpd,
3213 struct ib_qp_init_attr *attrs,
3214 struct ib_udata *udata)
3215 {
3216 struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device);
3217 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3218 struct qlnxr_qp *qp;
3219 int rc = 0;
3220 qlnx_host_t *ha;
3221
3222 ha = dev->ha;
3223
3224 QL_DPRINT12(ha, "enter\n");
3225
3226 rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata);
3227 if (rc) {
3228 QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc);
3229 return ERR_PTR(rc);
3230 }
3231
3232 QL_DPRINT12(ha, "called from %s, event_handle=%p,"
3233 " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
3234 (udata ? "user library" : "kernel"),
3235 attrs->event_handler, pd,
3236 get_qlnxr_cq(attrs->send_cq),
3237 get_qlnxr_cq(attrs->send_cq)->icid,
3238 get_qlnxr_cq(attrs->recv_cq),
3239 get_qlnxr_cq(attrs->recv_cq)->icid);
3240
3241 qp = qlnx_zalloc(sizeof(struct qlnxr_qp));
3242
3243 if (!qp) {
3244 QL_DPRINT11(ha, "kzalloc(qp) failed\n");
3245 return ERR_PTR(-ENOMEM);
3246 }
3247
3248 qlnxr_set_common_qp_params(dev, qp, pd, attrs);
3249
3250 if (attrs->qp_type == IB_QPT_GSI) {
3251 QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n");
3252 return qlnxr_create_gsi_qp(dev, attrs, qp);
3253 }
3254
3255 if (udata) {
3256 rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs);
3257
3258 if (rc) {
3259 QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n");
3260 goto err;
3261 }
3262 } else {
3263 rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs);
3264
3265 if (rc) {
3266 QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n");
3267 goto err;
3268 }
3269 }
3270
3271 qp->ibqp.qp_num = qp->qp_id;
3272
3273 rc = qlnxr_idr_add(dev, qp, qp->qp_id);
3274
3275 if (rc) {
3276 QL_DPRINT11(ha, "qlnxr_idr_add failed\n");
3277 goto err;
3278 }
3279
3280 QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp);
3281
3282 return &qp->ibqp;
3283 err:
3284 kfree(qp);
3285
3286 QL_DPRINT12(ha, "failed exit\n");
3287 return ERR_PTR(-EFAULT);
3288 }
3289
3290 static enum ib_qp_state
3291 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state)
3292 {
3293 enum ib_qp_state state = IB_QPS_ERR;
3294
3295 switch (qp_state) {
3296 case ECORE_ROCE_QP_STATE_RESET:
3297 state = IB_QPS_RESET;
3298 break;
3299
3300 case ECORE_ROCE_QP_STATE_INIT:
3301 state = IB_QPS_INIT;
3302 break;
3303
3304 case ECORE_ROCE_QP_STATE_RTR:
3305 state = IB_QPS_RTR;
3306 break;
3307
3308 case ECORE_ROCE_QP_STATE_RTS:
3309 state = IB_QPS_RTS;
3310 break;
3311
3312 case ECORE_ROCE_QP_STATE_SQD:
3313 state = IB_QPS_SQD;
3314 break;
3315
3316 case ECORE_ROCE_QP_STATE_ERR:
3317 state = IB_QPS_ERR;
3318 break;
3319
3320 case ECORE_ROCE_QP_STATE_SQE:
3321 state = IB_QPS_SQE;
3322 break;
3323 }
3324 return state;
3325 }
3326
3327 static enum ecore_roce_qp_state
3328 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state)
3329 {
3330 enum ecore_roce_qp_state ecore_qp_state;
3331
3332 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3333
3334 switch (qp_state) {
3335 case IB_QPS_RESET:
3336 ecore_qp_state = ECORE_ROCE_QP_STATE_RESET;
3337 break;
3338
3339 case IB_QPS_INIT:
3340 ecore_qp_state = ECORE_ROCE_QP_STATE_INIT;
3341 break;
3342
3343 case IB_QPS_RTR:
3344 ecore_qp_state = ECORE_ROCE_QP_STATE_RTR;
3345 break;
3346
3347 case IB_QPS_RTS:
3348 ecore_qp_state = ECORE_ROCE_QP_STATE_RTS;
3349 break;
3350
3351 case IB_QPS_SQD:
3352 ecore_qp_state = ECORE_ROCE_QP_STATE_SQD;
3353 break;
3354
3355 case IB_QPS_ERR:
3356 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3357 break;
3358
3359 default:
3360 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3361 break;
3362 }
3363
3364 return (ecore_qp_state);
3365 }
3366
3367 static void
3368 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph)
3369 {
3370 ecore_chain_reset(&qph->pbl);
3371 qph->prod = qph->cons = 0;
3372 qph->wqe_cons = 0;
3373 qph->db_data.data.value = cpu_to_le16(0);
3374
3375 return;
3376 }
3377
3378 static int
3379 qlnxr_update_qp_state(struct qlnxr_dev *dev,
3380 struct qlnxr_qp *qp,
3381 enum ecore_roce_qp_state new_state)
3382 {
3383 int status = 0;
3384 uint32_t reg_addr;
3385 struct ecore_dev *cdev;
3386 qlnx_host_t *ha;
3387
3388 ha = dev->ha;
3389 cdev = &ha->cdev;
3390
3391 QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n",
3392 qp, new_state, qp->state);
3393
3394 if (new_state == qp->state) {
3395 return 0;
3396 }
3397
3398 switch (qp->state) {
3399 case ECORE_ROCE_QP_STATE_RESET:
3400 switch (new_state) {
3401 case ECORE_ROCE_QP_STATE_INIT:
3402 qp->prev_wqe_size = 0;
3403 qlnxr_reset_qp_hwq_info(&qp->sq);
3404 if (!(qp->srq))
3405 qlnxr_reset_qp_hwq_info(&qp->rq);
3406 break;
3407 default:
3408 status = -EINVAL;
3409 break;
3410 };
3411 break;
3412 case ECORE_ROCE_QP_STATE_INIT:
3413 /* INIT->XXX */
3414 switch (new_state) {
3415 case ECORE_ROCE_QP_STATE_RTR:
3416 /* Update doorbell (in case post_recv was done before move to RTR) */
3417 if (qp->srq)
3418 break;
3419 wmb();
3420 //writel(qp->rq.db_data.raw, qp->rq.db);
3421 //if (QLNX_IS_IWARP(dev))
3422 // writel(qp->rq.iwarp_db2_data.raw,
3423 // qp->rq.iwarp_db2);
3424
3425 reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
3426 (uint8_t *)cdev->doorbells);
3427
3428 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
3429 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
3430
3431 if (QLNX_IS_IWARP(dev)) {
3432 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
3433 (uint8_t *)cdev->doorbells);
3434 bus_write_4(ha->pci_dbells, reg_addr,\
3435 qp->rq.iwarp_db2_data.raw);
3436 bus_barrier(ha->pci_dbells, 0, 0,\
3437 BUS_SPACE_BARRIER_READ);
3438 }
3439
3440
3441 mmiowb();
3442 break;
3443 case ECORE_ROCE_QP_STATE_ERR:
3444 /* TBD:flush qps... */
3445 break;
3446 default:
3447 /* invalid state change. */
3448 status = -EINVAL;
3449 break;
3450 };
3451 break;
3452 case ECORE_ROCE_QP_STATE_RTR:
3453 /* RTR->XXX */
3454 switch (new_state) {
3455 case ECORE_ROCE_QP_STATE_RTS:
3456 break;
3457 case ECORE_ROCE_QP_STATE_ERR:
3458 break;
3459 default:
3460 /* invalid state change. */
3461 status = -EINVAL;
3462 break;
3463 };
3464 break;
3465 case ECORE_ROCE_QP_STATE_RTS:
3466 /* RTS->XXX */
3467 switch (new_state) {
3468 case ECORE_ROCE_QP_STATE_SQD:
3469 break;
3470 case ECORE_ROCE_QP_STATE_ERR:
3471 break;
3472 default:
3473 /* invalid state change. */
3474 status = -EINVAL;
3475 break;
3476 };
3477 break;
3478 case ECORE_ROCE_QP_STATE_SQD:
3479 /* SQD->XXX */
3480 switch (new_state) {
3481 case ECORE_ROCE_QP_STATE_RTS:
3482 case ECORE_ROCE_QP_STATE_ERR:
3483 break;
3484 default:
3485 /* invalid state change. */
3486 status = -EINVAL;
3487 break;
3488 };
3489 break;
3490 case ECORE_ROCE_QP_STATE_ERR:
3491 /* ERR->XXX */
3492 switch (new_state) {
3493 case ECORE_ROCE_QP_STATE_RESET:
3494 if ((qp->rq.prod != qp->rq.cons) ||
3495 (qp->sq.prod != qp->sq.cons)) {
3496 QL_DPRINT11(ha,
3497 "Error->Reset with rq/sq "
3498 "not empty rq.prod=0x%x rq.cons=0x%x"
3499 " sq.prod=0x%x sq.cons=0x%x\n",
3500 qp->rq.prod, qp->rq.cons,
3501 qp->sq.prod, qp->sq.cons);
3502 status = -EINVAL;
3503 }
3504 break;
3505 default:
3506 status = -EINVAL;
3507 break;
3508 };
3509 break;
3510 default:
3511 status = -EINVAL;
3512 break;
3513 };
3514
3515 QL_DPRINT12(ha, "exit\n");
3516 return status;
3517 }
3518
3519 int
3520 qlnxr_modify_qp(struct ib_qp *ibqp,
3521 struct ib_qp_attr *attr,
3522 int attr_mask,
3523 struct ib_udata *udata)
3524 {
3525 int rc = 0;
3526 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3527 struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev);
3528 struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
3529 enum ib_qp_state old_qp_state, new_qp_state;
3530 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3531 qlnx_host_t *ha;
3532
3533 ha = dev->ha;
3534
3535 QL_DPRINT12(ha,
3536 "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n",
3537 qp, attr_mask, attr->qp_state, udata);
3538
3539 old_qp_state = qlnxr_get_ibqp_state(qp->state);
3540 if (attr_mask & IB_QP_STATE)
3541 new_qp_state = attr->qp_state;
3542 else
3543 new_qp_state = old_qp_state;
3544
3545 if (QLNX_IS_ROCE(dev)) {
3546 if (!ib_modify_qp_is_ok(old_qp_state,
3547 new_qp_state,
3548 ibqp->qp_type,
3549 attr_mask )) {
3550 QL_DPRINT12(ha,
3551 "invalid attribute mask=0x%x"
3552 " specified for qpn=0x%x of type=0x%x \n"
3553 " old_qp_state=0x%x, new_qp_state=0x%x\n",
3554 attr_mask, qp->qp_id, ibqp->qp_type,
3555 old_qp_state, new_qp_state);
3556 rc = -EINVAL;
3557 goto err;
3558 }
3559 }
3560 /* translate the masks... */
3561 if (attr_mask & IB_QP_STATE) {
3562 SET_FIELD(qp_params.modify_flags,
3563 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
3564 qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state);
3565 }
3566
3567 // TBD consider changing ecore to be a flag as well...
3568 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
3569 qp_params.sqd_async = true;
3570
3571 if (attr_mask & IB_QP_PKEY_INDEX) {
3572 SET_FIELD(qp_params.modify_flags,
3573 ECORE_ROCE_MODIFY_QP_VALID_PKEY,
3574 1);
3575 if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) {
3576 rc = -EINVAL;
3577 goto err;
3578 }
3579
3580 qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT;
3581 }
3582
3583 if (attr_mask & IB_QP_QKEY) {
3584 qp->qkey = attr->qkey;
3585 }
3586
3587 /* tbd consider splitting in ecore.. */
3588 if (attr_mask & IB_QP_ACCESS_FLAGS) {
3589 SET_FIELD(qp_params.modify_flags,
3590 ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
3591 qp_params.incoming_rdma_read_en =
3592 attr->qp_access_flags & IB_ACCESS_REMOTE_READ;
3593 qp_params.incoming_rdma_write_en =
3594 attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE;
3595 qp_params.incoming_atomic_en =
3596 attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC;
3597 }
3598
3599 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
3600 if (attr_mask & IB_QP_PATH_MTU) {
3601 if (attr->path_mtu < IB_MTU_256 ||
3602 attr->path_mtu > IB_MTU_4096) {
3603 QL_DPRINT12(ha,
3604 "Only MTU sizes of 256, 512, 1024,"
3605 " 2048 and 4096 are supported "
3606 " attr->path_mtu = [%d]\n",
3607 attr->path_mtu);
3608
3609 rc = -EINVAL;
3610 goto err;
3611 }
3612 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
3613 ib_mtu_enum_to_int(
3614 iboe_get_mtu(dev->ha->ifp->if_mtu)));
3615 }
3616
3617 if (qp->mtu == 0) {
3618 qp->mtu = ib_mtu_enum_to_int(
3619 iboe_get_mtu(dev->ha->ifp->if_mtu));
3620 QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n",
3621 qp->mtu);
3622 }
3623
3624 SET_FIELD(qp_params.modify_flags,
3625 ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR,
3626 1);
3627
3628 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
3629 qp_params.flow_label = attr->ah_attr.grh.flow_label;
3630 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
3631
3632 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
3633
3634 get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params);
3635
3636 rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr);
3637 if (rc)
3638 return rc;
3639
3640 qp_params.use_local_mac = true;
3641 memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
3642
3643 QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n",
3644 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
3645 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
3646 QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n",
3647 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
3648 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
3649 QL_DPRINT12(ha,
3650 "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n",
3651 qp_params.remote_mac_addr[0],
3652 qp_params.remote_mac_addr[1],
3653 qp_params.remote_mac_addr[2],
3654 qp_params.remote_mac_addr[3],
3655 qp_params.remote_mac_addr[4],
3656 qp_params.remote_mac_addr[5]);
3657
3658 qp_params.mtu = qp->mtu;
3659 }
3660
3661 if (qp_params.mtu == 0) {
3662 /* stay with current MTU */
3663 if (qp->mtu) {
3664 qp_params.mtu = qp->mtu;
3665 } else {
3666 qp_params.mtu = ib_mtu_enum_to_int(
3667 iboe_get_mtu(dev->ha->ifp->if_mtu));
3668 }
3669 }
3670
3671 if (attr_mask & IB_QP_TIMEOUT) {
3672 SET_FIELD(qp_params.modify_flags, \
3673 ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
3674
3675 qp_params.ack_timeout = attr->timeout;
3676 if (attr->timeout) {
3677 u32 temp;
3678
3679 /* 12.7.34 LOCAL ACK TIMEOUT
3680 * Value representing the transport (ACK) timeout for
3681 * use by the remote, expressed as (4.096 μS*2Local ACK
3682 * Timeout)
3683 */
3684 /* We use 1UL since the temporal value may be overflow
3685 * 32 bits
3686 */
3687 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
3688 qp_params.ack_timeout = temp; /* FW requires [msec] */
3689 }
3690 else
3691 qp_params.ack_timeout = 0; /* infinite */
3692 }
3693 if (attr_mask & IB_QP_RETRY_CNT) {
3694 SET_FIELD(qp_params.modify_flags,\
3695 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
3696 qp_params.retry_cnt = attr->retry_cnt;
3697 }
3698
3699 if (attr_mask & IB_QP_RNR_RETRY) {
3700 SET_FIELD(qp_params.modify_flags,
3701 ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT,
3702 1);
3703 qp_params.rnr_retry_cnt = attr->rnr_retry;
3704 }
3705
3706 if (attr_mask & IB_QP_RQ_PSN) {
3707 SET_FIELD(qp_params.modify_flags,
3708 ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN,
3709 1);
3710 qp_params.rq_psn = attr->rq_psn;
3711 qp->rq_psn = attr->rq_psn;
3712 }
3713
3714 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
3715 if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) {
3716 rc = -EINVAL;
3717 QL_DPRINT12(ha,
3718 "unsupported max_rd_atomic=%d, supported=%d\n",
3719 attr->max_rd_atomic,
3720 qattr->max_qp_req_rd_atomic_resc);
3721 goto err;
3722 }
3723
3724 SET_FIELD(qp_params.modify_flags,
3725 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ,
3726 1);
3727 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
3728 }
3729
3730 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
3731 SET_FIELD(qp_params.modify_flags,
3732 ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER,
3733 1);
3734 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
3735 }
3736
3737 if (attr_mask & IB_QP_SQ_PSN) {
3738 SET_FIELD(qp_params.modify_flags,
3739 ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN,
3740 1);
3741 qp_params.sq_psn = attr->sq_psn;
3742 qp->sq_psn = attr->sq_psn;
3743 }
3744
3745 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
3746 if (attr->max_dest_rd_atomic >
3747 qattr->max_qp_resp_rd_atomic_resc) {
3748 QL_DPRINT12(ha,
3749 "unsupported max_dest_rd_atomic=%d, "
3750 "supported=%d\n",
3751 attr->max_dest_rd_atomic,
3752 qattr->max_qp_resp_rd_atomic_resc);
3753
3754 rc = -EINVAL;
3755 goto err;
3756 }
3757
3758 SET_FIELD(qp_params.modify_flags,
3759 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP,
3760 1);
3761 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
3762 }
3763
3764 if (attr_mask & IB_QP_DEST_QPN) {
3765 SET_FIELD(qp_params.modify_flags,
3766 ECORE_ROCE_MODIFY_QP_VALID_DEST_QP,
3767 1);
3768
3769 qp_params.dest_qp = attr->dest_qp_num;
3770 qp->dest_qp_num = attr->dest_qp_num;
3771 }
3772
3773 /*
3774 * Update the QP state before the actual ramrod to prevent a race with
3775 * fast path. Modifying the QP state to error will cause the device to
3776 * flush the CQEs and while polling the flushed CQEs will considered as
3777 * a potential issue if the QP isn't in error state.
3778 */
3779 if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) &&
3780 (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR))
3781 qp->state = ECORE_ROCE_QP_STATE_ERR;
3782
3783 if (qp->qp_type != IB_QPT_GSI)
3784 rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
3785
3786 if (attr_mask & IB_QP_STATE) {
3787 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
3788 rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state);
3789 qp->state = qp_params.new_state;
3790 }
3791
3792 err:
3793 QL_DPRINT12(ha, "exit\n");
3794 return rc;
3795 }
3796
3797 static int
3798 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params)
3799 {
3800 int ib_qp_acc_flags = 0;
3801
3802 if (params->incoming_rdma_write_en)
3803 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
3804 if (params->incoming_rdma_read_en)
3805 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
3806 if (params->incoming_atomic_en)
3807 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
3808 if (true) /* FIXME -> local write ?? */
3809 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
3810
3811 return ib_qp_acc_flags;
3812 }
3813
3814 static enum ib_mtu
3815 qlnxr_mtu_int_to_enum(u16 mtu)
3816 {
3817 enum ib_mtu ib_mtu_size;
3818
3819 switch (mtu) {
3820 case 256:
3821 ib_mtu_size = IB_MTU_256;
3822 break;
3823
3824 case 512:
3825 ib_mtu_size = IB_MTU_512;
3826 break;
3827
3828 case 1024:
3829 ib_mtu_size = IB_MTU_1024;
3830 break;
3831
3832 case 2048:
3833 ib_mtu_size = IB_MTU_2048;
3834 break;
3835
3836 case 4096:
3837 ib_mtu_size = IB_MTU_4096;
3838 break;
3839
3840 default:
3841 ib_mtu_size = IB_MTU_1024;
3842 break;
3843 }
3844 return (ib_mtu_size);
3845 }
3846
3847 int
3848 qlnxr_query_qp(struct ib_qp *ibqp,
3849 struct ib_qp_attr *qp_attr,
3850 int attr_mask,
3851 struct ib_qp_init_attr *qp_init_attr)
3852 {
3853 int rc = 0;
3854 struct ecore_rdma_query_qp_out_params params;
3855 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3856 struct qlnxr_dev *dev = qp->dev;
3857 qlnx_host_t *ha;
3858
3859 ha = dev->ha;
3860
3861 QL_DPRINT12(ha, "enter\n");
3862
3863 memset(¶ms, 0, sizeof(params));
3864
3865 rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms);
3866 if (rc)
3867 goto err;
3868
3869 memset(qp_attr, 0, sizeof(*qp_attr));
3870 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
3871
3872 qp_attr->qp_state = qlnxr_get_ibqp_state(params.state);
3873 qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state);
3874
3875 /* In some cases in iWARP qelr will ask for the state only */
3876 if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) {
3877 QL_DPRINT11(ha, "only state requested\n");
3878 return 0;
3879 }
3880
3881 qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu);
3882 qp_attr->path_mig_state = IB_MIG_MIGRATED;
3883 qp_attr->rq_psn = params.rq_psn;
3884 qp_attr->sq_psn = params.sq_psn;
3885 qp_attr->dest_qp_num = params.dest_qp;
3886
3887 qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms);
3888
3889 QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x "
3890 "path_mtu = %d qp_access_flags = 0x%x\n",
3891 qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu,
3892 qp_attr->qp_access_flags);
3893
3894 qp_attr->cap.max_send_wr = qp->sq.max_wr;
3895 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
3896 qp_attr->cap.max_send_sge = qp->sq.max_sges;
3897 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
3898 qp_attr->cap.max_inline_data = qp->max_inline_data;
3899 qp_init_attr->cap = qp_attr->cap;
3900
3901 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0],
3902 sizeof(qp_attr->ah_attr.grh.dgid.raw));
3903
3904 qp_attr->ah_attr.grh.flow_label = params.flow_label;
3905 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
3906 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
3907 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
3908
3909 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
3910 qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */
3911 qp_attr->ah_attr.sl = 0;/* FIXME -> check this */
3912 qp_attr->timeout = params.timeout;
3913 qp_attr->rnr_retry = params.rnr_retry;
3914 qp_attr->retry_cnt = params.retry_cnt;
3915 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
3916 qp_attr->pkey_index = params.pkey_index;
3917 qp_attr->port_num = 1; /* FIXME -> check this */
3918 qp_attr->ah_attr.src_path_bits = 0;
3919 qp_attr->ah_attr.static_rate = 0;
3920 qp_attr->alt_pkey_index = 0;
3921 qp_attr->alt_port_num = 0;
3922 qp_attr->alt_timeout = 0;
3923 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
3924
3925 qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0;
3926 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
3927 qp_attr->max_rd_atomic = params.max_rd_atomic;
3928 qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0;
3929
3930 QL_DPRINT12(ha, "max_inline_data=%d\n",
3931 qp_attr->cap.max_inline_data);
3932
3933 err:
3934 QL_DPRINT12(ha, "exit\n");
3935 return rc;
3936 }
3937
3938 static void
3939 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
3940 {
3941 qlnx_host_t *ha;
3942
3943 ha = dev->ha;
3944
3945 QL_DPRINT12(ha, "enter\n");
3946
3947 if (qp->usq.umem)
3948 ib_umem_release(qp->usq.umem);
3949
3950 qp->usq.umem = NULL;
3951
3952 if (qp->urq.umem)
3953 ib_umem_release(qp->urq.umem);
3954
3955 qp->urq.umem = NULL;
3956
3957 QL_DPRINT12(ha, "exit\n");
3958 return;
3959 }
3960
3961 static void
3962 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
3963 {
3964 qlnx_host_t *ha;
3965
3966 ha = dev->ha;
3967
3968 QL_DPRINT12(ha, "enter\n");
3969
3970 if (qlnxr_qp_has_sq(qp)) {
3971 QL_DPRINT12(ha, "freeing SQ\n");
3972 ha->qlnxr_debug = 1;
3973 // ecore_chain_free(dev->cdev, &qp->sq.pbl);
3974 ha->qlnxr_debug = 0;
3975 kfree(qp->wqe_wr_id);
3976 }
3977
3978 if (qlnxr_qp_has_rq(qp)) {
3979 QL_DPRINT12(ha, "freeing RQ\n");
3980 ha->qlnxr_debug = 1;
3981 // ecore_chain_free(dev->cdev, &qp->rq.pbl);
3982 ha->qlnxr_debug = 0;
3983 kfree(qp->rqe_wr_id);
3984 }
3985
3986 QL_DPRINT12(ha, "exit\n");
3987 return;
3988 }
3989
3990 int
3991 qlnxr_free_qp_resources(struct qlnxr_dev *dev,
3992 struct qlnxr_qp *qp)
3993 {
3994 int rc = 0;
3995 qlnx_host_t *ha;
3996 struct ecore_rdma_destroy_qp_out_params d_out_params;
3997
3998 ha = dev->ha;
3999
4000 QL_DPRINT12(ha, "enter\n");
4001
4002 #if 0
4003 if (qp->qp_type != IB_QPT_GSI) {
4004 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4005 &d_out_params);
4006 if (rc)
4007 return rc;
4008 }
4009
4010 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4011 qlnxr_cleanup_user(dev, qp);
4012 else
4013 qlnxr_cleanup_kernel(dev, qp);
4014 #endif
4015
4016 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4017 qlnxr_cleanup_user(dev, qp);
4018 else
4019 qlnxr_cleanup_kernel(dev, qp);
4020
4021 if (qp->qp_type != IB_QPT_GSI) {
4022 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4023 &d_out_params);
4024 if (rc)
4025 return rc;
4026 }
4027
4028 QL_DPRINT12(ha, "exit\n");
4029 return 0;
4030 }
4031
4032 int
4033 qlnxr_destroy_qp(struct ib_qp *ibqp)
4034 {
4035 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4036 struct qlnxr_dev *dev = qp->dev;
4037 int rc = 0;
4038 struct ib_qp_attr attr;
4039 int attr_mask = 0;
4040 qlnx_host_t *ha;
4041
4042 ha = dev->ha;
4043
4044 QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type);
4045
4046 qp->destroyed = 1;
4047
4048 if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET |
4049 ECORE_ROCE_QP_STATE_ERR |
4050 ECORE_ROCE_QP_STATE_INIT))) {
4051 attr.qp_state = IB_QPS_ERR;
4052 attr_mask |= IB_QP_STATE;
4053
4054 /* change the QP state to ERROR */
4055 qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL);
4056 }
4057
4058 if (qp->qp_type == IB_QPT_GSI)
4059 qlnxr_destroy_gsi_qp(dev);
4060
4061 qp->sig = ~qp->sig;
4062
4063 qlnxr_free_qp_resources(dev, qp);
4064
4065 if (atomic_dec_and_test(&qp->refcnt)) {
4066 /* TODO: only for iWARP? */
4067 qlnxr_idr_remove(dev, qp->qp_id);
4068 kfree(qp);
4069 }
4070
4071 QL_DPRINT12(ha, "exit\n");
4072 return rc;
4073 }
4074
4075 static inline int
4076 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq)
4077 {
4078 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
4079 }
4080
4081 static int
4082 sge_data_len(struct ib_sge *sg_list, int num_sge)
4083 {
4084 int i, len = 0;
4085 for (i = 0; i < num_sge; i++)
4086 len += sg_list[i].length;
4087 return len;
4088 }
4089
4090 static void
4091 swap_wqe_data64(u64 *p)
4092 {
4093 int i;
4094
4095 for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
4096 *p = cpu_to_be64(cpu_to_le64(*p));
4097 }
4098
4099 static u32
4100 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev,
4101 struct qlnxr_qp *qp,
4102 u8 *wqe_size,
4103 const struct ib_send_wr *wr,
4104 const struct ib_send_wr **bad_wr,
4105 u8 *bits,
4106 u8 bit)
4107 {
4108 int i, seg_siz;
4109 char *seg_prt, *wqe;
4110 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
4111 qlnx_host_t *ha;
4112
4113 ha = dev->ha;
4114
4115 QL_DPRINT12(ha, "enter[%d]\n", data_size);
4116
4117 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
4118 QL_DPRINT12(ha,
4119 "Too much inline data in WR:[%d, %d]\n",
4120 data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE);
4121 *bad_wr = wr;
4122 return 0;
4123 }
4124
4125 if (!data_size)
4126 return data_size;
4127
4128 /* set the bit */
4129 *bits |= bit;
4130
4131 seg_prt = wqe = NULL;
4132 seg_siz = 0;
4133
4134 /* copy data inline */
4135 for (i = 0; i < wr->num_sge; i++) {
4136 u32 len = wr->sg_list[i].length;
4137 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
4138
4139 while (len > 0) {
4140 u32 cur;
4141
4142 /* new segment required */
4143 if (!seg_siz) {
4144 wqe = (char *)ecore_chain_produce(&qp->sq.pbl);
4145 seg_prt = wqe;
4146 seg_siz = sizeof(struct rdma_sq_common_wqe);
4147 (*wqe_size)++;
4148 }
4149
4150 /* calculate currently allowed length */
4151 cur = MIN(len, seg_siz);
4152
4153 memcpy(seg_prt, src, cur);
4154
4155 /* update segment variables */
4156 seg_prt += cur;
4157 seg_siz -= cur;
4158 /* update sge variables */
4159 src += cur;
4160 len -= cur;
4161
4162 /* swap fully-completed segments */
4163 if (!seg_siz)
4164 swap_wqe_data64((u64 *)wqe);
4165 }
4166 }
4167
4168 /* swap last not completed segment */
4169 if (seg_siz)
4170 swap_wqe_data64((u64 *)wqe);
4171
4172 QL_DPRINT12(ha, "exit\n");
4173 return data_size;
4174 }
4175
4176 static u32
4177 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp,
4178 u8 *wqe_size, const struct ib_send_wr *wr)
4179 {
4180 int i;
4181 u32 data_size = 0;
4182 qlnx_host_t *ha;
4183
4184 ha = dev->ha;
4185
4186 QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge);
4187
4188 for (i = 0; i < wr->num_sge; i++) {
4189 struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl);
4190
4191 TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr);
4192 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
4193 sge->length = cpu_to_le32(wr->sg_list[i].length);
4194 data_size += wr->sg_list[i].length;
4195 }
4196
4197 if (wqe_size)
4198 *wqe_size += wr->num_sge;
4199
4200 QL_DPRINT12(ha, "exit data_size = %d\n", data_size);
4201 return data_size;
4202 }
4203
4204 static u32
4205 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev,
4206 struct qlnxr_qp *qp,
4207 struct rdma_sq_rdma_wqe_1st *rwqe,
4208 struct rdma_sq_rdma_wqe_2nd *rwqe2,
4209 const struct ib_send_wr *wr,
4210 const struct ib_send_wr **bad_wr)
4211 {
4212 qlnx_host_t *ha;
4213 u32 ret = 0;
4214
4215 ha = dev->ha;
4216
4217 QL_DPRINT12(ha, "enter\n");
4218
4219 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
4220 TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr);
4221
4222 if (wr->send_flags & IB_SEND_INLINE) {
4223 u8 flags = 0;
4224 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
4225 return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size,
4226 wr, bad_wr, &rwqe->flags, flags);
4227 }
4228
4229 ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr);
4230
4231 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4232
4233 return (ret);
4234 }
4235
4236 static u32
4237 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev,
4238 struct qlnxr_qp *qp,
4239 struct rdma_sq_send_wqe *swqe,
4240 struct rdma_sq_send_wqe *swqe2,
4241 const struct ib_send_wr *wr,
4242 const struct ib_send_wr **bad_wr)
4243 {
4244 qlnx_host_t *ha;
4245 u32 ret = 0;
4246
4247 ha = dev->ha;
4248
4249 QL_DPRINT12(ha, "enter\n");
4250
4251 memset(swqe2, 0, sizeof(*swqe2));
4252
4253 if (wr->send_flags & IB_SEND_INLINE) {
4254 u8 flags = 0;
4255 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
4256 return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size,
4257 wr, bad_wr, &swqe->flags, flags);
4258 }
4259
4260 ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr);
4261
4262 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4263
4264 return (ret);
4265 }
4266
4267 static void
4268 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info)
4269 {
4270 qlnx_host_t *ha;
4271
4272 ha = dev->ha;
4273
4274 int work = info->completed - info->completed_handled - 1;
4275
4276 QL_DPRINT12(ha, "enter [%d]\n", work);
4277
4278 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
4279 struct qlnxr_pbl *pbl;
4280
4281 /* Free all the page list that are possible to be freed
4282 * (all the ones that were invalidated), under the assumption
4283 * that if an FMR was completed successfully that means that
4284 * if there was an invalidate operation before it also ended
4285 */
4286 pbl = list_first_entry(&info->inuse_pbl_list,
4287 struct qlnxr_pbl,
4288 list_entry);
4289 list_del(&pbl->list_entry);
4290 list_add_tail(&pbl->list_entry, &info->free_pbl_list);
4291 info->completed_handled++;
4292 }
4293
4294 QL_DPRINT12(ha, "exit\n");
4295 return;
4296 }
4297
4298 #if __FreeBSD_version >= 1102000
4299
4300 static int qlnxr_prepare_reg(struct qlnxr_qp *qp,
4301 struct rdma_sq_fmr_wqe_1st *fwqe1,
4302 const struct ib_reg_wr *wr)
4303 {
4304 struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr);
4305 struct rdma_sq_fmr_wqe_2nd *fwqe2;
4306
4307 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl);
4308 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
4309 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
4310 fwqe1->l_key = wr->key;
4311
4312 fwqe2->access_ctrl = 0;
4313
4314 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
4315 !!(wr->access & IB_ACCESS_REMOTE_READ));
4316 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
4317 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
4318 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
4319 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
4320 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
4321 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
4322 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
4323 fwqe2->fmr_ctrl = 0;
4324
4325 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4326 ilog2(mr->ibmr.page_size) - 12);
4327
4328 fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */
4329 fwqe2->length_lo = mr->ibmr.length;
4330 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
4331 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
4332
4333 qp->wqe_wr_id[qp->sq.prod].mr = mr;
4334
4335 return 0;
4336 }
4337
4338 #else
4339
4340 static void
4341 build_frmr_pbes(struct qlnxr_dev *dev, const struct ib_send_wr *wr,
4342 struct mr_info *info)
4343 {
4344 int i;
4345 u64 buf_addr = 0;
4346 int num_pbes, total_num_pbes = 0;
4347 struct regpair *pbe;
4348 struct qlnxr_pbl *pbl_tbl = info->pbl_table;
4349 struct qlnxr_pbl_info *pbl_info = &info->pbl_info;
4350 qlnx_host_t *ha;
4351
4352 ha = dev->ha;
4353
4354 QL_DPRINT12(ha, "enter\n");
4355
4356 pbe = (struct regpair *)pbl_tbl->va;
4357 num_pbes = 0;
4358
4359 for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) {
4360 buf_addr = wr->wr.fast_reg.page_list->page_list[i];
4361 pbe->lo = cpu_to_le32((u32)buf_addr);
4362 pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr));
4363
4364 num_pbes += 1;
4365 pbe++;
4366 total_num_pbes++;
4367
4368 if (total_num_pbes == pbl_info->num_pbes)
4369 return;
4370
4371 /* if the given pbl is full storing the pbes,
4372 * move to next pbl.
4373 */
4374 if (num_pbes ==
4375 (pbl_info->pbl_size / sizeof(u64))) {
4376 pbl_tbl++;
4377 pbe = (struct regpair *)pbl_tbl->va;
4378 num_pbes = 0;
4379 }
4380 }
4381 QL_DPRINT12(ha, "exit\n");
4382
4383 return;
4384 }
4385
4386 static int
4387 qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info)
4388 {
4389 int rc = 0;
4390 qlnx_host_t *ha;
4391
4392 ha = dev->ha;
4393
4394 QL_DPRINT12(ha, "enter\n");
4395
4396 if (info->completed == 0) {
4397 //DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n");
4398 /* first fmr */
4399 return 0;
4400 }
4401
4402 qlnx_handle_completed_mrs(dev, info);
4403
4404 list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list);
4405
4406 if (list_empty(&info->free_pbl_list)) {
4407 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info,
4408 GFP_ATOMIC);
4409 } else {
4410 info->pbl_table = list_first_entry(&info->free_pbl_list,
4411 struct qlnxr_pbl,
4412 list_entry);
4413 list_del(&info->pbl_table->list_entry);
4414 }
4415
4416 if (!info->pbl_table)
4417 rc = -ENOMEM;
4418
4419 QL_DPRINT12(ha, "exit\n");
4420 return rc;
4421 }
4422
4423 static inline int
4424 qlnxr_prepare_fmr(struct qlnxr_qp *qp,
4425 struct rdma_sq_fmr_wqe_1st *fwqe1,
4426 const struct ib_send_wr *wr)
4427 {
4428 struct qlnxr_dev *dev = qp->dev;
4429 u64 fbo;
4430 struct qlnxr_fast_reg_page_list *frmr_list =
4431 get_qlnxr_frmr_list(wr->wr.fast_reg.page_list);
4432 struct rdma_sq_fmr_wqe *fwqe2 =
4433 (struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl);
4434 int rc = 0;
4435 qlnx_host_t *ha;
4436
4437 ha = dev->ha;
4438
4439 QL_DPRINT12(ha, "enter\n");
4440
4441 if (wr->wr.fast_reg.page_list_len == 0)
4442 BUG();
4443
4444 rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info);
4445 if (rc)
4446 return rc;
4447
4448 fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start);
4449 fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start);
4450 fwqe1->l_key = wr->wr.fast_reg.rkey;
4451
4452 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ,
4453 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ));
4454 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE,
4455 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE));
4456 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC,
4457 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC));
4458 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1);
4459 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE,
4460 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE));
4461
4462 fwqe2->fmr_ctrl = 0;
4463
4464 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4465 ilog2(1 << wr->wr.fast_reg.page_shift) - 12);
4466 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0);
4467
4468 fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */
4469 fwqe2->length_lo = wr->wr.fast_reg.length;
4470 fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa);
4471 fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa);
4472
4473 /* produce another wqe for fwqe3 */
4474 ecore_chain_produce(&qp->sq.pbl);
4475
4476 fbo = wr->wr.fast_reg.iova_start -
4477 (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK);
4478
4479 QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x"
4480 " length = %x pbl_addr %x:%x\n",
4481 wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey,
4482 fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo,
4483 fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo);
4484
4485 build_frmr_pbes(dev, wr, &frmr_list->info);
4486
4487 qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list;
4488
4489 QL_DPRINT12(ha, "exit\n");
4490 return 0;
4491 }
4492
4493 #endif /* #if __FreeBSD_version >= 1102000 */
4494
4495 static enum ib_wc_opcode
4496 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
4497 {
4498 switch (opcode) {
4499 case IB_WR_RDMA_WRITE:
4500 case IB_WR_RDMA_WRITE_WITH_IMM:
4501 return IB_WC_RDMA_WRITE;
4502 case IB_WR_SEND_WITH_IMM:
4503 case IB_WR_SEND:
4504 case IB_WR_SEND_WITH_INV:
4505 return IB_WC_SEND;
4506 case IB_WR_RDMA_READ:
4507 return IB_WC_RDMA_READ;
4508 case IB_WR_ATOMIC_CMP_AND_SWP:
4509 return IB_WC_COMP_SWAP;
4510 case IB_WR_ATOMIC_FETCH_AND_ADD:
4511 return IB_WC_FETCH_ADD;
4512
4513 #if __FreeBSD_version >= 1102000
4514 case IB_WR_REG_MR:
4515 return IB_WC_REG_MR;
4516 #else
4517 case IB_WR_FAST_REG_MR:
4518 return IB_WC_FAST_REG_MR;
4519 #endif /* #if __FreeBSD_version >= 1102000 */
4520
4521 case IB_WR_LOCAL_INV:
4522 return IB_WC_LOCAL_INV;
4523 default:
4524 return IB_WC_SEND;
4525 }
4526 }
4527 static inline bool
4528 qlnxr_can_post_send(struct qlnxr_qp *qp, const struct ib_send_wr *wr)
4529 {
4530 int wq_is_full, err_wr, pbl_is_full;
4531 struct qlnxr_dev *dev = qp->dev;
4532 qlnx_host_t *ha;
4533
4534 ha = dev->ha;
4535
4536 QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr);
4537
4538 /* prevent SQ overflow and/or processing of a bad WR */
4539 err_wr = wr->num_sge > qp->sq.max_sges;
4540 wq_is_full = qlnxr_wq_is_full(&qp->sq);
4541 pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) <
4542 QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
4543 if (wq_is_full || err_wr || pbl_is_full) {
4544 if (wq_is_full &&
4545 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) {
4546 qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL;
4547
4548 QL_DPRINT12(ha,
4549 "error: WQ is full. Post send on QP failed"
4550 " (this error appears only once) "
4551 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4552 qp, wr, qp->err_bitmap);
4553 }
4554
4555 if (err_wr &&
4556 !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) {
4557 qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR;
4558
4559 QL_DPRINT12(ha,
4560 "error: WQ is bad. Post send on QP failed"
4561 " (this error appears only once) "
4562 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4563 qp, wr, qp->err_bitmap);
4564 }
4565
4566 if (pbl_is_full &&
4567 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) {
4568 qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL;
4569
4570 QL_DPRINT12(ha,
4571 "error: WQ PBL is full. Post send on QP failed"
4572 " (this error appears only once) "
4573 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4574 qp, wr, qp->err_bitmap);
4575 }
4576 return false;
4577 }
4578 QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr);
4579 return true;
4580 }
4581
4582 int
4583 qlnxr_post_send(struct ib_qp *ibqp,
4584 const struct ib_send_wr *wr,
4585 const struct ib_send_wr **bad_wr)
4586 {
4587 struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device);
4588 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4589 unsigned long flags;
4590 int status = 0, rc = 0;
4591 bool comp;
4592 qlnx_host_t *ha;
4593 uint32_t reg_addr;
4594
4595 *bad_wr = NULL;
4596 ha = dev->ha;
4597
4598 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
4599 ibqp, wr, bad_wr);
4600
4601 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
4602 return -EINVAL;
4603
4604 if (qp->qp_type == IB_QPT_GSI)
4605 return qlnxr_gsi_post_send(ibqp, wr, bad_wr);
4606
4607 spin_lock_irqsave(&qp->q_lock, flags);
4608
4609 if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) &&
4610 (qp->state != ECORE_ROCE_QP_STATE_ERR) &&
4611 (qp->state != ECORE_ROCE_QP_STATE_SQD)) {
4612 spin_unlock_irqrestore(&qp->q_lock, flags);
4613 *bad_wr = wr;
4614 QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n",
4615 qp->icid, qp->state);
4616 return -EINVAL;
4617 }
4618
4619 if (!wr) {
4620 QL_DPRINT11(ha, "Got an empty post send???\n");
4621 }
4622
4623 while (wr) {
4624 struct rdma_sq_common_wqe *wqe;
4625 struct rdma_sq_send_wqe *swqe;
4626 struct rdma_sq_send_wqe *swqe2;
4627 struct rdma_sq_rdma_wqe_1st *rwqe;
4628 struct rdma_sq_rdma_wqe_2nd *rwqe2;
4629 struct rdma_sq_local_inv_wqe *iwqe;
4630 struct rdma_sq_atomic_wqe *awqe1;
4631 struct rdma_sq_atomic_wqe *awqe2;
4632 struct rdma_sq_atomic_wqe *awqe3;
4633 struct rdma_sq_fmr_wqe_1st *fwqe1;
4634
4635 if (!qlnxr_can_post_send(qp, wr)) {
4636 status = -ENOMEM;
4637 *bad_wr = wr;
4638 break;
4639 }
4640
4641 wqe = ecore_chain_produce(&qp->sq.pbl);
4642
4643 qp->wqe_wr_id[qp->sq.prod].signaled =
4644 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
4645
4646 /* common fields */
4647 wqe->flags = 0;
4648 wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK <<
4649 RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT);
4650
4651 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \
4652 !!(wr->send_flags & IB_SEND_SOLICITED));
4653
4654 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) ||
4655 (qp->signaled);
4656
4657 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
4658 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \
4659 !!(wr->send_flags & IB_SEND_FENCE));
4660
4661 wqe->prev_wqe_size = qp->prev_wqe_size;
4662
4663 qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode);
4664
4665 switch (wr->opcode) {
4666 case IB_WR_SEND_WITH_IMM:
4667
4668 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
4669 swqe = (struct rdma_sq_send_wqe *)wqe;
4670 swqe->wqe_size = 2;
4671 swqe2 = (struct rdma_sq_send_wqe *)
4672 ecore_chain_produce(&qp->sq.pbl);
4673 swqe->inv_key_or_imm_data =
4674 cpu_to_le32(wr->ex.imm_data);
4675 swqe->length = cpu_to_le32(
4676 qlnxr_prepare_sq_send_data(dev,
4677 qp, swqe, swqe2, wr,
4678 bad_wr));
4679
4680 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4681 qp->prev_wqe_size = swqe->wqe_size;
4682 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4683
4684 QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n",
4685 swqe->length, wr->ex.imm_data);
4686
4687 break;
4688
4689 case IB_WR_SEND:
4690
4691 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
4692 swqe = (struct rdma_sq_send_wqe *)wqe;
4693
4694 swqe->wqe_size = 2;
4695 swqe2 = (struct rdma_sq_send_wqe *)
4696 ecore_chain_produce(&qp->sq.pbl);
4697 swqe->length = cpu_to_le32(
4698 qlnxr_prepare_sq_send_data(dev,
4699 qp, swqe, swqe2, wr,
4700 bad_wr));
4701 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4702 qp->prev_wqe_size = swqe->wqe_size;
4703 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4704
4705 QL_DPRINT12(ha, "SEND w/o IMM length = %d\n",
4706 swqe->length);
4707
4708 break;
4709
4710 case IB_WR_SEND_WITH_INV:
4711
4712 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
4713 swqe = (struct rdma_sq_send_wqe *)wqe;
4714 swqe2 = (struct rdma_sq_send_wqe *)
4715 ecore_chain_produce(&qp->sq.pbl);
4716 swqe->wqe_size = 2;
4717 swqe->inv_key_or_imm_data =
4718 cpu_to_le32(wr->ex.invalidate_rkey);
4719 swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev,
4720 qp, swqe, swqe2, wr, bad_wr));
4721 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4722 qp->prev_wqe_size = swqe->wqe_size;
4723 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4724
4725 QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n",
4726 swqe->length);
4727 break;
4728
4729 case IB_WR_RDMA_WRITE_WITH_IMM:
4730
4731 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
4732 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4733
4734 rwqe->wqe_size = 2;
4735 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
4736 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4737 ecore_chain_produce(&qp->sq.pbl);
4738 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4739 qp, rwqe, rwqe2, wr, bad_wr));
4740 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4741 qp->prev_wqe_size = rwqe->wqe_size;
4742 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4743
4744 QL_DPRINT12(ha,
4745 "RDMA WRITE w/ IMM length = %d imm data=%x\n",
4746 rwqe->length, rwqe->imm_data);
4747
4748 break;
4749
4750 case IB_WR_RDMA_WRITE:
4751
4752 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
4753 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4754
4755 rwqe->wqe_size = 2;
4756 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4757 ecore_chain_produce(&qp->sq.pbl);
4758 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4759 qp, rwqe, rwqe2, wr, bad_wr));
4760 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4761 qp->prev_wqe_size = rwqe->wqe_size;
4762 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4763
4764 QL_DPRINT12(ha,
4765 "RDMA WRITE w/o IMM length = %d\n",
4766 rwqe->length);
4767
4768 break;
4769
4770 case IB_WR_RDMA_READ_WITH_INV:
4771
4772 QL_DPRINT12(ha,
4773 "RDMA READ WITH INVALIDATE not supported\n");
4774
4775 *bad_wr = wr;
4776 rc = -EINVAL;
4777
4778 break;
4779
4780 case IB_WR_RDMA_READ:
4781
4782 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
4783 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4784
4785 rwqe->wqe_size = 2;
4786 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4787 ecore_chain_produce(&qp->sq.pbl);
4788 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4789 qp, rwqe, rwqe2, wr, bad_wr));
4790
4791 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4792 qp->prev_wqe_size = rwqe->wqe_size;
4793 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4794
4795 QL_DPRINT12(ha, "RDMA READ length = %d\n",
4796 rwqe->length);
4797
4798 break;
4799
4800 case IB_WR_ATOMIC_CMP_AND_SWP:
4801 case IB_WR_ATOMIC_FETCH_AND_ADD:
4802
4803 QL_DPRINT12(ha,
4804 "ATOMIC operation = %s\n",
4805 ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ?
4806 "IB_WR_ATOMIC_CMP_AND_SWP" :
4807 "IB_WR_ATOMIC_FETCH_AND_ADD"));
4808
4809 awqe1 = (struct rdma_sq_atomic_wqe *)wqe;
4810 awqe1->prev_wqe_size = 4;
4811
4812 awqe2 = (struct rdma_sq_atomic_wqe *)
4813 ecore_chain_produce(&qp->sq.pbl);
4814
4815 TYPEPTR_ADDR_SET(awqe2, remote_va, \
4816 atomic_wr(wr)->remote_addr);
4817
4818 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
4819
4820 awqe3 = (struct rdma_sq_atomic_wqe *)
4821 ecore_chain_produce(&qp->sq.pbl);
4822
4823 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
4824 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
4825 TYPEPTR_ADDR_SET(awqe3, swap_data,
4826 atomic_wr(wr)->compare_add);
4827 } else {
4828 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
4829 TYPEPTR_ADDR_SET(awqe3, swap_data,
4830 atomic_wr(wr)->swap);
4831 TYPEPTR_ADDR_SET(awqe3, cmp_data,
4832 atomic_wr(wr)->compare_add);
4833 }
4834
4835 qlnxr_prepare_sq_sges(dev, qp, NULL, wr);
4836
4837 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size;
4838 qp->prev_wqe_size = awqe1->prev_wqe_size;
4839
4840 break;
4841
4842 case IB_WR_LOCAL_INV:
4843
4844 QL_DPRINT12(ha,
4845 "INVALIDATE length (IB_WR_LOCAL_INV)\n");
4846
4847 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
4848 iwqe->prev_wqe_size = 1;
4849
4850 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
4851 iwqe->inv_l_key = wr->ex.invalidate_rkey;
4852 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size;
4853 qp->prev_wqe_size = iwqe->prev_wqe_size;
4854
4855 break;
4856
4857 #if __FreeBSD_version >= 1102000
4858
4859 case IB_WR_REG_MR:
4860
4861 QL_DPRINT12(ha, "IB_WR_REG_MR\n");
4862
4863 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4864 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4865 fwqe1->wqe_size = 2;
4866
4867 rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr));
4868 if (rc) {
4869 QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc);
4870 *bad_wr = wr;
4871 break;
4872 }
4873
4874 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
4875 qp->prev_wqe_size = fwqe1->wqe_size;
4876
4877 break;
4878 #else
4879 case IB_WR_FAST_REG_MR:
4880
4881 QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n");
4882
4883 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4884 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4885 fwqe1->prev_wqe_size = 3;
4886
4887 rc = qlnxr_prepare_fmr(qp, fwqe1, wr);
4888
4889 if (rc) {
4890 QL_DPRINT12(ha,
4891 "FAST_MR (IB_WR_FAST_REG_MR) failed"
4892 " rc = %d\n", rc);
4893 *bad_wr = wr;
4894 break;
4895 }
4896
4897 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size;
4898 qp->prev_wqe_size = fwqe1->prev_wqe_size;
4899
4900 break;
4901 #endif /* #if __FreeBSD_version >= 1102000 */
4902
4903 default:
4904
4905 QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode);
4906
4907 rc = -EINVAL;
4908 *bad_wr = wr;
4909 break;
4910 }
4911
4912 if (*bad_wr) {
4913 /*
4914 * restore prod to its position before this WR was processed
4915 */
4916 ecore_chain_set_prod(&qp->sq.pbl,
4917 le16_to_cpu(qp->sq.db_data.data.value),
4918 wqe);
4919 /* restore prev_wqe_size */
4920 qp->prev_wqe_size = wqe->prev_wqe_size;
4921 status = rc;
4922
4923 QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr);
4924 break; /* out of the loop */
4925 }
4926
4927 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
4928
4929 qlnxr_inc_sw_prod(&qp->sq);
4930
4931 qp->sq.db_data.data.value++;
4932
4933 wr = wr->next;
4934 }
4935
4936 /* Trigger doorbell
4937 * If there was a failure in the first WR then it will be triggered in
4938 * vane. However this is not harmful (as long as the producer value is
4939 * unchanged). For performance reasons we avoid checking for this
4940 * redundant doorbell.
4941 */
4942 wmb();
4943 //writel(qp->sq.db_data.raw, qp->sq.db);
4944
4945 reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells);
4946 bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw);
4947 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
4948
4949 mmiowb();
4950
4951 spin_unlock_irqrestore(&qp->q_lock, flags);
4952
4953 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
4954 ibqp, wr, bad_wr);
4955
4956 return status;
4957 }
4958
4959 static u32
4960 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq)
4961 {
4962 u32 used;
4963
4964 /* Calculate number of elements used based on producer
4965 * count and consumer count and subtract it from max
4966 * work request supported so that we get elements left.
4967 */
4968 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
4969
4970 return hw_srq->max_wr - used;
4971 }
4972
4973 int
4974 qlnxr_post_recv(struct ib_qp *ibqp,
4975 const struct ib_recv_wr *wr,
4976 const struct ib_recv_wr **bad_wr)
4977 {
4978 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4979 struct qlnxr_dev *dev = qp->dev;
4980 unsigned long flags;
4981 int status = 0;
4982 qlnx_host_t *ha;
4983 uint32_t reg_addr;
4984
4985 ha = dev->ha;
4986
4987 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
4988 return -EINVAL;
4989
4990 QL_DPRINT12(ha, "enter\n");
4991
4992 if (qp->qp_type == IB_QPT_GSI) {
4993 QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n");
4994 return qlnxr_gsi_post_recv(ibqp, wr, bad_wr);
4995 }
4996
4997 if (qp->srq) {
4998 QL_DPRINT11(ha, "qp->srq [%p]"
4999 " QP is associated with SRQ, cannot post RQ buffers\n",
5000 qp->srq);
5001 return -EINVAL;
5002 }
5003
5004 spin_lock_irqsave(&qp->q_lock, flags);
5005
5006 if (qp->state == ECORE_ROCE_QP_STATE_RESET) {
5007 spin_unlock_irqrestore(&qp->q_lock, flags);
5008 *bad_wr = wr;
5009
5010 QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n");
5011
5012 return -EINVAL;
5013 }
5014
5015 while (wr) {
5016 int i;
5017
5018 if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) <
5019 QLNXR_MAX_RQE_ELEMENTS_PER_RQE) ||
5020 (wr->num_sge > qp->rq.max_sges)) {
5021 status = -ENOMEM;
5022 *bad_wr = wr;
5023 break;
5024 }
5025 for (i = 0; i < wr->num_sge; i++) {
5026 u32 flags = 0;
5027 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5028
5029 /* first one must include the number of SGE in the list */
5030 if (!i)
5031 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge);
5032
5033 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey);
5034
5035 RQ_SGE_SET(rqe, wr->sg_list[i].addr, \
5036 wr->sg_list[i].length, flags);
5037 }
5038 /* Special case of no sges. FW requires between 1-4 sges...
5039 * in this case we need to post 1 sge with length zero. this is
5040 * because rdma write with immediate consumes an RQ. */
5041 if (!wr->num_sge) {
5042 u32 flags = 0;
5043 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5044
5045 /* first one must include the number of SGE in the list */
5046 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
5047 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
5048
5049 //RQ_SGE_SET(rqe, 0, 0, flags);
5050 rqe->addr.hi = 0;
5051 rqe->addr.lo = 0;
5052
5053 rqe->length = 0;
5054 rqe->flags = cpu_to_le32(flags);
5055
5056 i = 1;
5057 }
5058
5059 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
5060 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
5061
5062 qlnxr_inc_sw_prod(&qp->rq);
5063
5064 wmb();
5065
5066 qp->rq.db_data.data.value++;
5067
5068 // writel(qp->rq.db_data.raw, qp->rq.db);
5069 mmiowb();
5070 // if (QLNX_IS_IWARP(dev)) {
5071 // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
5072 // mmiowb(); /* for second doorbell */
5073 // }
5074
5075 reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
5076 (uint8_t *)ha->cdev.doorbells);
5077
5078 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
5079 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
5080
5081 if (QLNX_IS_IWARP(dev)) {
5082 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
5083 (uint8_t *)ha->cdev.doorbells);
5084 bus_write_4(ha->pci_dbells, reg_addr, \
5085 qp->rq.iwarp_db2_data.raw);
5086 bus_barrier(ha->pci_dbells, 0, 0, \
5087 BUS_SPACE_BARRIER_READ);
5088 }
5089
5090 wr = wr->next;
5091 }
5092
5093 spin_unlock_irqrestore(&qp->q_lock, flags);
5094
5095 QL_DPRINT12(ha, "exit status = 0x%x\n", status);
5096
5097 return status;
5098 }
5099
5100 /* In fmr we need to increase the number of fmr completed counter for the fmr
5101 * algorithm determining whether we can free a pbl or not.
5102 * we need to perform this whether the work request was signaled or not. for
5103 * this purpose we call this function from the condition that checks if a wr
5104 * should be skipped, to make sure we don't miss it ( possibly this fmr
5105 * operation was not signalted)
5106 */
5107 static inline void
5108 qlnxr_chk_if_fmr(struct qlnxr_qp *qp)
5109 {
5110 #if __FreeBSD_version >= 1102000
5111
5112 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
5113 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5114 #else
5115 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR)
5116 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5117
5118 #endif /* #if __FreeBSD_version >= 1102000 */
5119 }
5120
5121 static int
5122 process_req(struct qlnxr_dev *dev,
5123 struct qlnxr_qp *qp,
5124 struct qlnxr_cq *cq,
5125 int num_entries,
5126 struct ib_wc *wc,
5127 u16 hw_cons,
5128 enum ib_wc_status status,
5129 int force)
5130 {
5131 u16 cnt = 0;
5132 qlnx_host_t *ha = dev->ha;
5133
5134 QL_DPRINT12(ha, "enter\n");
5135
5136 while (num_entries && qp->sq.wqe_cons != hw_cons) {
5137 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
5138 qlnxr_chk_if_fmr(qp);
5139 /* skip WC */
5140 goto next_cqe;
5141 }
5142
5143 /* fill WC */
5144 wc->status = status;
5145 wc->vendor_err = 0;
5146 wc->wc_flags = 0;
5147 wc->src_qp = qp->id;
5148 wc->qp = &qp->ibqp;
5149
5150 // common section
5151 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
5152 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
5153
5154 switch (wc->opcode) {
5155 case IB_WC_RDMA_WRITE:
5156
5157 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
5158
5159 QL_DPRINT12(ha,
5160 "opcode = IB_WC_RDMA_WRITE bytes = %d\n",
5161 qp->wqe_wr_id[qp->sq.cons].bytes_len);
5162 break;
5163
5164 case IB_WC_COMP_SWAP:
5165 case IB_WC_FETCH_ADD:
5166 wc->byte_len = 8;
5167 break;
5168
5169 #if __FreeBSD_version >= 1102000
5170 case IB_WC_REG_MR:
5171 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5172 break;
5173 #else
5174 case IB_WC_FAST_REG_MR:
5175 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5176 break;
5177 #endif /* #if __FreeBSD_version >= 1102000 */
5178
5179 case IB_WC_RDMA_READ:
5180 case IB_WC_SEND:
5181
5182 QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode);
5183 break;
5184 default:
5185 ;//DP_ERR("TBD ERROR");
5186 }
5187
5188 num_entries--;
5189 wc++;
5190 cnt++;
5191 next_cqe:
5192 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
5193 ecore_chain_consume(&qp->sq.pbl);
5194 qlnxr_inc_sw_cons(&qp->sq);
5195 }
5196
5197 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5198 return cnt;
5199 }
5200
5201 static int
5202 qlnxr_poll_cq_req(struct qlnxr_dev *dev,
5203 struct qlnxr_qp *qp,
5204 struct qlnxr_cq *cq,
5205 int num_entries,
5206 struct ib_wc *wc,
5207 struct rdma_cqe_requester *req)
5208 {
5209 int cnt = 0;
5210 qlnx_host_t *ha = dev->ha;
5211
5212 QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status);
5213
5214 switch (req->status) {
5215 case RDMA_CQE_REQ_STS_OK:
5216
5217 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5218 IB_WC_SUCCESS, 0);
5219 break;
5220
5221 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
5222
5223 if (qp->state != ECORE_ROCE_QP_STATE_ERR)
5224 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5225 IB_WC_WR_FLUSH_ERR, 1);
5226 break;
5227
5228 default: /* other errors case */
5229
5230 /* process all WQE before the cosumer */
5231 qp->state = ECORE_ROCE_QP_STATE_ERR;
5232 cnt = process_req(dev, qp, cq, num_entries, wc,
5233 req->sq_cons - 1, IB_WC_SUCCESS, 0);
5234 wc += cnt;
5235 /* if we have extra WC fill it with actual error info */
5236
5237 if (cnt < num_entries) {
5238 enum ib_wc_status wc_status;
5239
5240 switch (req->status) {
5241 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
5242 wc_status = IB_WC_BAD_RESP_ERR;
5243 break;
5244 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
5245 wc_status = IB_WC_LOC_LEN_ERR;
5246 break;
5247 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
5248 wc_status = IB_WC_LOC_QP_OP_ERR;
5249 break;
5250 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
5251 wc_status = IB_WC_LOC_PROT_ERR;
5252 break;
5253 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
5254 wc_status = IB_WC_MW_BIND_ERR;
5255 break;
5256 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
5257 wc_status = IB_WC_REM_INV_REQ_ERR;
5258 break;
5259 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
5260 wc_status = IB_WC_REM_ACCESS_ERR;
5261 break;
5262 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
5263 wc_status = IB_WC_REM_OP_ERR;
5264 break;
5265 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
5266 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
5267 break;
5268 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
5269 wc_status = IB_WC_RETRY_EXC_ERR;
5270 break;
5271 default:
5272 wc_status = IB_WC_GENERAL_ERR;
5273 }
5274
5275 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
5276 wc_status, 1 /* force use of WC */);
5277 }
5278 }
5279
5280 QL_DPRINT12(ha, "exit cnt = %d\n", cnt);
5281 return cnt;
5282 }
5283
5284 static void
5285 __process_resp_one(struct qlnxr_dev *dev,
5286 struct qlnxr_qp *qp,
5287 struct qlnxr_cq *cq,
5288 struct ib_wc *wc,
5289 struct rdma_cqe_responder *resp,
5290 u64 wr_id)
5291 {
5292 enum ib_wc_status wc_status = IB_WC_SUCCESS;
5293 #if __FreeBSD_version < 1102000
5294 u8 flags;
5295 #endif
5296 qlnx_host_t *ha = dev->ha;
5297
5298 QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n",
5299 qp, resp->status);
5300
5301 wc->opcode = IB_WC_RECV;
5302 wc->wc_flags = 0;
5303
5304 switch (resp->status) {
5305 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
5306 wc_status = IB_WC_LOC_ACCESS_ERR;
5307 break;
5308
5309 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
5310 wc_status = IB_WC_LOC_LEN_ERR;
5311 break;
5312
5313 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
5314 wc_status = IB_WC_LOC_QP_OP_ERR;
5315 break;
5316
5317 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
5318 wc_status = IB_WC_LOC_PROT_ERR;
5319 break;
5320
5321 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
5322 wc_status = IB_WC_MW_BIND_ERR;
5323 break;
5324
5325 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
5326 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
5327 break;
5328
5329 case RDMA_CQE_RESP_STS_OK:
5330
5331 #if __FreeBSD_version >= 1102000
5332 if (resp->flags & QLNXR_RESP_IMM) {
5333 wc->ex.imm_data =
5334 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5335 wc->wc_flags |= IB_WC_WITH_IMM;
5336
5337 if (resp->flags & QLNXR_RESP_RDMA)
5338 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5339
5340 if (resp->flags & QLNXR_RESP_INV) {
5341 QL_DPRINT11(ha,
5342 "Invalid flags QLNXR_RESP_INV [0x%x]"
5343 "qp = %p qp->id = 0x%x cq = %p"
5344 " cq->icid = 0x%x\n",
5345 resp->flags, qp, qp->id, cq, cq->icid );
5346 }
5347 } else if (resp->flags & QLNXR_RESP_INV) {
5348 wc->ex.imm_data =
5349 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5350 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
5351
5352 if (resp->flags & QLNXR_RESP_RDMA) {
5353 QL_DPRINT11(ha,
5354 "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5355 "qp = %p qp->id = 0x%x cq = %p"
5356 " cq->icid = 0x%x\n",
5357 resp->flags, qp, qp->id, cq, cq->icid );
5358 }
5359 } else if (resp->flags & QLNXR_RESP_RDMA) {
5360 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5361 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5362 resp->flags, qp, qp->id, cq, cq->icid );
5363 }
5364 #else
5365 wc_status = IB_WC_SUCCESS;
5366 wc->byte_len = le32_to_cpu(resp->length);
5367
5368 flags = resp->flags & QLNXR_RESP_RDMA_IMM;
5369
5370 switch (flags) {
5371 case QLNXR_RESP_RDMA_IMM:
5372 /* update opcode */
5373 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5374 /* fall to set imm data */
5375 case QLNXR_RESP_IMM:
5376 wc->ex.imm_data =
5377 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5378 wc->wc_flags |= IB_WC_WITH_IMM;
5379 break;
5380 case QLNXR_RESP_RDMA:
5381 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5382 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5383 resp->flags, qp, qp->id, cq, cq->icid );
5384 break;
5385 default:
5386 /* valid configuration, but nothing todo here */
5387 ;
5388 }
5389 #endif /* #if __FreeBSD_version >= 1102000 */
5390
5391 break;
5392 default:
5393 wc_status = IB_WC_GENERAL_ERR;
5394 }
5395
5396 /* fill WC */
5397 wc->status = wc_status;
5398 wc->vendor_err = 0;
5399 wc->src_qp = qp->id;
5400 wc->qp = &qp->ibqp;
5401 wc->wr_id = wr_id;
5402
5403 QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status);
5404
5405 return;
5406 }
5407
5408 static int
5409 process_resp_one_srq(struct qlnxr_dev *dev,
5410 struct qlnxr_qp *qp,
5411 struct qlnxr_cq *cq,
5412 struct ib_wc *wc,
5413 struct rdma_cqe_responder *resp)
5414 {
5415 struct qlnxr_srq *srq = qp->srq;
5416 u64 wr_id;
5417 qlnx_host_t *ha = dev->ha;
5418
5419 QL_DPRINT12(ha, "enter\n");
5420
5421 wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo);
5422
5423 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5424 wc->status = IB_WC_WR_FLUSH_ERR;
5425 wc->vendor_err = 0;
5426 wc->wr_id = wr_id;
5427 wc->byte_len = 0;
5428 wc->src_qp = qp->id;
5429 wc->qp = &qp->ibqp;
5430 wc->wr_id = wr_id;
5431 } else {
5432 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
5433 }
5434
5435 /* PBL is maintained in case of WR granularity.
5436 * So increment WR consumer after consuming WR
5437 */
5438 srq->hw_srq.wr_cons_cnt++;
5439
5440 QL_DPRINT12(ha, "exit\n");
5441 return 1;
5442 }
5443
5444 static int
5445 process_resp_one(struct qlnxr_dev *dev,
5446 struct qlnxr_qp *qp,
5447 struct qlnxr_cq *cq,
5448 struct ib_wc *wc,
5449 struct rdma_cqe_responder *resp)
5450 {
5451 qlnx_host_t *ha = dev->ha;
5452 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5453
5454 QL_DPRINT12(ha, "enter\n");
5455
5456 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
5457
5458 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5459 ecore_chain_consume(&qp->rq.pbl);
5460 qlnxr_inc_sw_cons(&qp->rq);
5461
5462 QL_DPRINT12(ha, "exit\n");
5463 return 1;
5464 }
5465
5466 static int
5467 process_resp_flush(struct qlnxr_qp *qp,
5468 int num_entries,
5469 struct ib_wc *wc,
5470 u16 hw_cons)
5471 {
5472 u16 cnt = 0;
5473 qlnx_host_t *ha = qp->dev->ha;
5474
5475 QL_DPRINT12(ha, "enter\n");
5476
5477 while (num_entries && qp->rq.wqe_cons != hw_cons) {
5478 /* fill WC */
5479 wc->status = IB_WC_WR_FLUSH_ERR;
5480 wc->vendor_err = 0;
5481 wc->wc_flags = 0;
5482 wc->src_qp = qp->id;
5483 wc->byte_len = 0;
5484 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5485 wc->qp = &qp->ibqp;
5486 num_entries--;
5487 wc++;
5488 cnt++;
5489 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5490 ecore_chain_consume(&qp->rq.pbl);
5491 qlnxr_inc_sw_cons(&qp->rq);
5492 }
5493
5494 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5495 return cnt;
5496 }
5497
5498 static void
5499 try_consume_resp_cqe(struct qlnxr_cq *cq,
5500 struct qlnxr_qp *qp,
5501 struct rdma_cqe_responder *resp,
5502 int *update)
5503 {
5504 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
5505 consume_cqe(cq);
5506 *update |= 1;
5507 }
5508 }
5509
5510 static int
5511 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev,
5512 struct qlnxr_qp *qp,
5513 struct qlnxr_cq *cq,
5514 int num_entries,
5515 struct ib_wc *wc,
5516 struct rdma_cqe_responder *resp,
5517 int *update)
5518 {
5519 int cnt;
5520 qlnx_host_t *ha = dev->ha;
5521
5522 QL_DPRINT12(ha, "enter\n");
5523
5524 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
5525 consume_cqe(cq);
5526 *update |= 1;
5527
5528 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5529 return cnt;
5530 }
5531
5532 static int
5533 qlnxr_poll_cq_resp(struct qlnxr_dev *dev,
5534 struct qlnxr_qp *qp,
5535 struct qlnxr_cq *cq,
5536 int num_entries,
5537 struct ib_wc *wc,
5538 struct rdma_cqe_responder *resp,
5539 int *update)
5540 {
5541 int cnt;
5542 qlnx_host_t *ha = dev->ha;
5543
5544 QL_DPRINT12(ha, "enter\n");
5545
5546 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5547 cnt = process_resp_flush(qp, num_entries, wc,
5548 resp->rq_cons);
5549 try_consume_resp_cqe(cq, qp, resp, update);
5550 } else {
5551 cnt = process_resp_one(dev, qp, cq, wc, resp);
5552 consume_cqe(cq);
5553 *update |= 1;
5554 }
5555
5556 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5557 return cnt;
5558 }
5559
5560 static void
5561 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp,
5562 struct rdma_cqe_requester *req, int *update)
5563 {
5564 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
5565 consume_cqe(cq);
5566 *update |= 1;
5567 }
5568 }
5569
5570 static void
5571 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags)
5572 {
5573 uint64_t reg_addr;
5574 qlnx_host_t *ha = dev->ha;
5575
5576 QL_DPRINT12(ha, "enter\n");
5577
5578 wmb();
5579 cq->db.data.agg_flags = flags;
5580 cq->db.data.value = cpu_to_le32(cons);
5581
5582 reg_addr = (uint64_t)((uint8_t *)cq->db_addr -
5583 (uint8_t *)(ha->cdev.doorbells));
5584
5585 bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw);
5586 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
5587
5588 QL_DPRINT12(ha, "exit\n");
5589 return;
5590
5591 //#ifdef __LP64__
5592 // writeq(cq->db.raw, cq->db_addr);
5593 //#else
5594 /* Note that since the FW allows 64 bit write only, in 32bit systems
5595 * the value of db_addr must be low enough. This is currently not
5596 * enforced.
5597 */
5598 // writel(cq->db.raw & 0xffffffff, cq->db_addr);
5599 // mmiowb();
5600 //#endif
5601 }
5602
5603 static int
5604 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe)
5605 {
5606 struct rdma_cqe_requester *resp_cqe = &cqe->req;
5607 return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) ==
5608 cq->pbl_toggle;
5609 }
5610
5611 int
5612 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
5613 {
5614 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
5615 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
5616 int done = 0;
5617 union rdma_cqe *cqe = cq->latest_cqe;
5618 int update = 0;
5619 u32 old_cons, new_cons;
5620 unsigned long flags;
5621 qlnx_host_t *ha = dev->ha;
5622
5623 QL_DPRINT12(ha, "enter\n");
5624
5625 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5626 return -EINVAL;
5627
5628 if (cq->destroyed) {
5629 QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n",
5630 cq, cq->icid);
5631 return 0;
5632 }
5633
5634 if (cq->cq_type == QLNXR_CQ_TYPE_GSI)
5635 return qlnxr_gsi_poll_cq(ibcq, num_entries, wc);
5636
5637 spin_lock_irqsave(&cq->cq_lock, flags);
5638
5639 old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5640
5641 while (num_entries && is_valid_cqe(cq, cqe)) {
5642 int cnt = 0;
5643 struct qlnxr_qp *qp;
5644 struct rdma_cqe_requester *resp_cqe;
5645 enum rdma_cqe_type cqe_type;
5646
5647 /* prevent speculative reads of any field of CQE */
5648 rmb();
5649
5650 resp_cqe = &cqe->req;
5651 qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi,
5652 resp_cqe->qp_handle.lo);
5653
5654 if (!qp) {
5655 QL_DPRINT11(ha, "qp = NULL\n");
5656 break;
5657 }
5658
5659 wc->qp = &qp->ibqp;
5660
5661 cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
5662
5663 switch (cqe_type) {
5664 case RDMA_CQE_TYPE_REQUESTER:
5665 cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries,
5666 wc, &cqe->req);
5667 try_consume_req_cqe(cq, qp, &cqe->req, &update);
5668 break;
5669 case RDMA_CQE_TYPE_RESPONDER_RQ:
5670 cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries,
5671 wc, &cqe->resp, &update);
5672 break;
5673 case RDMA_CQE_TYPE_RESPONDER_SRQ:
5674 cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries,
5675 wc, &cqe->resp, &update);
5676 break;
5677 case RDMA_CQE_TYPE_INVALID:
5678 default:
5679 QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type);
5680 break;
5681 }
5682 num_entries -= cnt;
5683 wc += cnt;
5684 done += cnt;
5685
5686 cqe = cq->latest_cqe;
5687 }
5688 new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5689
5690 cq->cq_cons += new_cons - old_cons;
5691
5692 if (update) {
5693 /* doorbell notifies abount latest VALID entry,
5694 * but chain already point to the next INVALID one
5695 */
5696 doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags);
5697 QL_DPRINT12(ha, "cq = %p cons = 0x%x "
5698 "arm_flags = 0x%x db.icid = 0x%x\n", cq,
5699 (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid);
5700 }
5701
5702 spin_unlock_irqrestore(&cq->cq_lock, flags);
5703
5704 QL_DPRINT12(ha, "exit\n");
5705
5706 return done;
5707 }
5708
5709 int
5710 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
5711 {
5712 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
5713 unsigned long sflags;
5714 struct qlnxr_dev *dev;
5715 qlnx_host_t *ha;
5716
5717 dev = get_qlnxr_dev((ibcq->device));
5718 ha = dev->ha;
5719
5720 QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x "
5721 "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq,
5722 flags, cq, cq->cq_cons, cq->cq_type);
5723
5724 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5725 return -EINVAL;
5726
5727 if (cq->destroyed) {
5728 QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n",
5729 cq, cq->icid);
5730 return -EINVAL;
5731 }
5732
5733 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) {
5734 return 0;
5735 }
5736
5737 spin_lock_irqsave(&cq->cq_lock, sflags);
5738
5739 cq->arm_flags = 0;
5740
5741 if (flags & IB_CQ_SOLICITED) {
5742 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
5743 }
5744 if (flags & IB_CQ_NEXT_COMP) {
5745 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
5746 }
5747
5748 doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags);
5749
5750 spin_unlock_irqrestore(&cq->cq_lock, sflags);
5751
5752 QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags);
5753 return 0;
5754 }
5755
5756 static struct qlnxr_mr *
5757 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len)
5758 {
5759 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
5760 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
5761 struct qlnxr_mr *mr;
5762 int rc = -ENOMEM;
5763 qlnx_host_t *ha;
5764
5765 ha = dev->ha;
5766
5767 QL_DPRINT12(ha, "enter ibpd = %p pd = %p "
5768 " pd_id = %d max_page_list_len = %d\n",
5769 ibpd, pd, pd->pd_id, max_page_list_len);
5770
5771 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
5772 if (!mr) {
5773 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
5774 return ERR_PTR(rc);
5775 }
5776
5777 mr->dev = dev;
5778 mr->type = QLNXR_MR_FRMR;
5779
5780 rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len,
5781 1 /* allow dual layer pbl */);
5782 if (rc) {
5783 QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n");
5784 goto err0;
5785 }
5786
5787 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
5788 if (rc) {
5789 QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n");
5790 goto err0;
5791 }
5792
5793 /* index only, 18 bit long, lkey = itid << 8 | key */
5794 mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR;
5795 mr->hw_mr.key = 0;
5796 mr->hw_mr.pd = pd->pd_id;
5797 mr->hw_mr.local_read = 1;
5798 mr->hw_mr.local_write = 0;
5799 mr->hw_mr.remote_read = 0;
5800 mr->hw_mr.remote_write = 0;
5801 mr->hw_mr.remote_atomic = 0;
5802 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
5803 mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */
5804 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
5805 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
5806 mr->hw_mr.fbo = 0;
5807 mr->hw_mr.length = 0;
5808 mr->hw_mr.vaddr = 0;
5809 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
5810 mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */
5811 mr->hw_mr.dma_mr = false;
5812
5813 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
5814 if (rc) {
5815 QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n");
5816 goto err1;
5817 }
5818
5819 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
5820 mr->ibmr.rkey = mr->ibmr.lkey;
5821
5822 QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n",
5823 mr, mr->ibmr.lkey);
5824
5825 return mr;
5826
5827 err1:
5828 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
5829 err0:
5830 kfree(mr);
5831
5832 QL_DPRINT12(ha, "exit\n");
5833
5834 return ERR_PTR(rc);
5835 }
5836
5837 #if __FreeBSD_version >= 1102000
5838
5839 struct ib_mr *
5840 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg)
5841 {
5842 struct qlnxr_dev *dev;
5843 struct qlnxr_mr *mr;
5844 qlnx_host_t *ha;
5845
5846 dev = get_qlnxr_dev(ibpd->device);
5847 ha = dev->ha;
5848
5849 QL_DPRINT12(ha, "enter\n");
5850
5851 if (mr_type != IB_MR_TYPE_MEM_REG)
5852 return ERR_PTR(-EINVAL);
5853
5854 mr = __qlnxr_alloc_mr(ibpd, max_num_sg);
5855
5856 if (IS_ERR(mr))
5857 return ERR_PTR(-EINVAL);
5858
5859 QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr);
5860
5861 return &mr->ibmr;
5862 }
5863
5864 static int
5865 qlnxr_set_page(struct ib_mr *ibmr, u64 addr)
5866 {
5867 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5868 struct qlnxr_pbl *pbl_table;
5869 struct regpair *pbe;
5870 struct qlnxr_dev *dev;
5871 qlnx_host_t *ha;
5872 u32 pbes_in_page;
5873
5874 dev = mr->dev;
5875 ha = dev->ha;
5876
5877 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
5878 QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages);
5879 return -ENOMEM;
5880 }
5881
5882 QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages,
5883 ((void *)addr));
5884
5885 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
5886 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
5887 pbe = (struct regpair *)pbl_table->va;
5888 pbe += mr->npages % pbes_in_page;
5889 pbe->lo = cpu_to_le32((u32)addr);
5890 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
5891
5892 mr->npages++;
5893
5894 QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages,
5895 ((void *)addr));
5896 return 0;
5897 }
5898
5899 int
5900 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
5901 int sg_nents, unsigned int *sg_offset)
5902 {
5903 int ret;
5904 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5905 qlnx_host_t *ha;
5906
5907 if (mr == NULL)
5908 return (-1);
5909
5910 if (mr->dev == NULL)
5911 return (-1);
5912
5913 ha = mr->dev->ha;
5914
5915 QL_DPRINT12(ha, "enter\n");
5916
5917 mr->npages = 0;
5918 qlnx_handle_completed_mrs(mr->dev, &mr->info);
5919
5920 ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page);
5921
5922 QL_DPRINT12(ha, "exit ret = %d\n", ret);
5923
5924 return (ret);
5925 }
5926
5927 #else
5928
5929 struct ib_mr *
5930 qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len)
5931 {
5932 struct qlnxr_dev *dev;
5933 struct qlnxr_mr *mr;
5934 qlnx_host_t *ha;
5935 struct ib_mr *ibmr = NULL;
5936
5937 dev = get_qlnxr_dev((ibpd->device));
5938 ha = dev->ha;
5939
5940 QL_DPRINT12(ha, "enter\n");
5941
5942 mr = __qlnxr_alloc_mr(ibpd, max_page_list_len);
5943
5944 if (IS_ERR(mr)) {
5945 ibmr = ERR_PTR(-EINVAL);
5946 } else {
5947 ibmr = &mr->ibmr;
5948 }
5949
5950 QL_DPRINT12(ha, "exit %p\n", ibmr);
5951 return (ibmr);
5952 }
5953
5954 void
5955 qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list)
5956 {
5957 struct qlnxr_fast_reg_page_list *frmr_list;
5958
5959 frmr_list = get_qlnxr_frmr_list(page_list);
5960
5961 free_mr_info(frmr_list->dev, &frmr_list->info);
5962
5963 kfree(frmr_list->ibfrpl.page_list);
5964 kfree(frmr_list);
5965
5966 return;
5967 }
5968
5969 struct ib_fast_reg_page_list *
5970 qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len)
5971 {
5972 struct qlnxr_fast_reg_page_list *frmr_list = NULL;
5973 struct qlnxr_dev *dev;
5974 int size = page_list_len * sizeof(u64);
5975 int rc = -ENOMEM;
5976 qlnx_host_t *ha;
5977
5978 dev = get_qlnxr_dev(ibdev);
5979 ha = dev->ha;
5980
5981 QL_DPRINT12(ha, "enter\n");
5982
5983 frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL);
5984 if (!frmr_list) {
5985 QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n");
5986 goto err;
5987 }
5988
5989 frmr_list->dev = dev;
5990 frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL);
5991 if (!frmr_list->ibfrpl.page_list) {
5992 QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n");
5993 goto err0;
5994 }
5995
5996 rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len,
5997 1 /* allow dual layer pbl */);
5998 if (rc)
5999 goto err1;
6000
6001 QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl);
6002
6003 return &frmr_list->ibfrpl;
6004
6005 err1:
6006 kfree(frmr_list->ibfrpl.page_list);
6007 err0:
6008 kfree(frmr_list);
6009 err:
6010 QL_DPRINT12(ha, "exit with error\n");
6011
6012 return ERR_PTR(rc);
6013 }
6014
6015 static int
6016 qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6017 int buf_cnt, uint64_t *total_size)
6018 {
6019 u64 size = 0;
6020
6021 *total_size = 0;
6022
6023 if (!buf_cnt || buf_list == NULL) {
6024 QL_DPRINT11(ha,
6025 "failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt);
6026 return (-1);
6027 }
6028
6029 size = buf_list->size;
6030
6031 if (!size) {
6032 QL_DPRINT11(ha,
6033 "failed buf_list = %p buf_cnt = %d"
6034 " buf_list->size = 0\n", buf_list, buf_cnt);
6035 return (-1);
6036 }
6037
6038 while (buf_cnt) {
6039 *total_size += buf_list->size;
6040
6041 if (buf_list->size != size) {
6042 QL_DPRINT11(ha,
6043 "failed buf_list = %p buf_cnt = %d"
6044 " all buffers should have same size\n",
6045 buf_list, buf_cnt);
6046 return (-1);
6047 }
6048
6049 buf_list++;
6050 buf_cnt--;
6051 }
6052 return (0);
6053 }
6054
6055 static size_t
6056 qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6057 int buf_cnt)
6058 {
6059 int i;
6060 size_t num_pages = 0;
6061 u64 size;
6062
6063 for (i = 0; i < buf_cnt; i++) {
6064 size = 0;
6065 while (size < buf_list->size) {
6066 size += PAGE_SIZE;
6067 num_pages++;
6068 }
6069 buf_list++;
6070 }
6071 return (num_pages);
6072 }
6073
6074 static void
6075 qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev,
6076 struct ib_phys_buf *buf_list, int buf_cnt,
6077 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
6078 {
6079 struct regpair *pbe;
6080 struct qlnxr_pbl *pbl_tbl;
6081 int pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
6082 qlnx_host_t *ha;
6083 int i;
6084 u64 pbe_addr;
6085
6086 ha = dev->ha;
6087
6088 QL_DPRINT12(ha, "enter\n");
6089
6090 if (!pbl_info) {
6091 QL_DPRINT11(ha, "PBL_INFO not initialized\n");
6092 return;
6093 }
6094
6095 if (!pbl_info->num_pbes) {
6096 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
6097 return;
6098 }
6099
6100 /* If we have a two layered pbl, the first pbl points to the rest
6101 * of the pbls and the first entry lays on the second pbl in the table
6102 */
6103 if (pbl_info->two_layered)
6104 pbl_tbl = &pbl[1];
6105 else
6106 pbl_tbl = pbl;
6107
6108 pbe = (struct regpair *)pbl_tbl->va;
6109 if (!pbe) {
6110 QL_DPRINT12(ha, "pbe is NULL\n");
6111 return;
6112 }
6113
6114 pbe_cnt = 0;
6115
6116 for (i = 0; i < buf_cnt; i++) {
6117 pages = buf_list->size >> PAGE_SHIFT;
6118
6119 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
6120 /* store the page address in pbe */
6121
6122 pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt);
6123
6124 pbe->lo = cpu_to_le32((u32)pbe_addr);
6125 pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32)));
6126
6127 QL_DPRINT12(ha, "Populate pbl table:"
6128 " pbe->addr=0x%x:0x%x "
6129 " pbe_cnt = %d total_num_pbes=%d"
6130 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
6131 total_num_pbes, pbe);
6132
6133 pbe_cnt ++;
6134 total_num_pbes ++;
6135 pbe++;
6136
6137 if (total_num_pbes == pbl_info->num_pbes)
6138 return;
6139
6140 /* if the given pbl is full storing the pbes,
6141 * move to next pbl. */
6142
6143 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
6144 pbl_tbl++;
6145 pbe = (struct regpair *)pbl_tbl->va;
6146 pbe_cnt = 0;
6147 }
6148 }
6149 buf_list++;
6150 }
6151 QL_DPRINT12(ha, "exit\n");
6152 return;
6153 }
6154
6155 struct ib_mr *
6156 qlnxr_reg_kernel_mr(struct ib_pd *ibpd,
6157 struct ib_phys_buf *buf_list,
6158 int buf_cnt, int acc, u64 *iova_start)
6159 {
6160 int rc = -ENOMEM;
6161 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
6162 struct qlnxr_mr *mr;
6163 struct qlnxr_pd *pd;
6164 qlnx_host_t *ha;
6165 size_t num_pages = 0;
6166 uint64_t length;
6167
6168 ha = dev->ha;
6169
6170 QL_DPRINT12(ha, "enter\n");
6171
6172 pd = get_qlnxr_pd(ibpd);
6173
6174 QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d,"
6175 " iova_start = %p, acc = %d\n",
6176 pd->pd_id, buf_list, buf_cnt, iova_start, acc);
6177
6178 //if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
6179 // QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&"
6180 // " !(acc & IB_ACCESS_LOCAL_WRITE))\n");
6181 // return ERR_PTR(-EINVAL);
6182 //}
6183
6184 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
6185 if (!mr) {
6186 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
6187 return ERR_PTR(rc);
6188 }
6189
6190 mr->type = QLNXR_MR_KERNEL;
6191 mr->iova_start = iova_start;
6192
6193 rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length);
6194 if (rc)
6195 goto err0;
6196
6197 num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt);
6198 if (!num_pages)
6199 goto err0;
6200
6201 rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1);
6202 if (rc) {
6203 QL_DPRINT11(ha,
6204 "qlnxr_init_mr_info failed [%d]\n", rc);
6205 goto err1;
6206 }
6207
6208 qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table,
6209 &mr->info.pbl_info);
6210
6211 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
6212
6213 if (rc) {
6214 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
6215 goto err1;
6216 }
6217
6218 /* index only, 18 bit long, lkey = itid << 8 | key */
6219 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
6220 mr->hw_mr.key = 0;
6221 mr->hw_mr.pd = pd->pd_id;
6222 mr->hw_mr.local_read = 1;
6223 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
6224 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
6225 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
6226 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
6227 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
6228 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
6229 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
6230 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
6231 mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */
6232
6233 mr->hw_mr.fbo = 0;
6234
6235 mr->hw_mr.length = length;
6236 mr->hw_mr.vaddr = (uint64_t)iova_start;
6237 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
6238 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
6239 mr->hw_mr.dma_mr = false;
6240
6241 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
6242 if (rc) {
6243 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
6244 goto err2;
6245 }
6246
6247 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6248 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
6249 mr->hw_mr.remote_atomic)
6250 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6251
6252 QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey);
6253
6254 return (&mr->ibmr);
6255
6256 err2:
6257 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
6258 err1:
6259 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
6260 err0:
6261 kfree(mr);
6262
6263 QL_DPRINT12(ha, "exit [%d]\n", rc);
6264 return (ERR_PTR(rc));
6265 }
6266
6267 #endif /* #if __FreeBSD_version >= 1102000 */
6268
6269 struct ib_ah *
6270 #if __FreeBSD_version >= 1102000
6271 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
6272 struct ib_udata *udata)
6273 #else
6274 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
6275 #endif /* #if __FreeBSD_version >= 1102000 */
6276 {
6277 struct qlnxr_dev *dev;
6278 qlnx_host_t *ha;
6279 struct qlnxr_ah *ah;
6280
6281 dev = get_qlnxr_dev((ibpd->device));
6282 ha = dev->ha;
6283
6284 QL_DPRINT12(ha, "in create_ah\n");
6285
6286 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
6287 if (!ah) {
6288 QL_DPRINT12(ha, "no address handle can be allocated\n");
6289 return ERR_PTR(-ENOMEM);
6290 }
6291
6292 ah->attr = *attr;
6293
6294 return &ah->ibah;
6295 }
6296
6297 int
6298 qlnxr_destroy_ah(struct ib_ah *ibah)
6299 {
6300 struct qlnxr_dev *dev;
6301 qlnx_host_t *ha;
6302 struct qlnxr_ah *ah = get_qlnxr_ah(ibah);
6303
6304 dev = get_qlnxr_dev((ibah->device));
6305 ha = dev->ha;
6306
6307 QL_DPRINT12(ha, "in destroy_ah\n");
6308
6309 kfree(ah);
6310 return 0;
6311 }
6312
6313 int
6314 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6315 {
6316 struct qlnxr_dev *dev;
6317 qlnx_host_t *ha;
6318
6319 dev = get_qlnxr_dev((ibah->device));
6320 ha = dev->ha;
6321 QL_DPRINT12(ha, "Query AH not supported\n");
6322 return -EINVAL;
6323 }
6324
6325 int
6326 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6327 {
6328 struct qlnxr_dev *dev;
6329 qlnx_host_t *ha;
6330
6331 dev = get_qlnxr_dev((ibah->device));
6332 ha = dev->ha;
6333 QL_DPRINT12(ha, "Modify AH not supported\n");
6334 return -ENOSYS;
6335 }
6336
6337 #if __FreeBSD_version >= 1102000
6338 int
6339 qlnxr_process_mad(struct ib_device *ibdev,
6340 int process_mad_flags,
6341 u8 port_num,
6342 const struct ib_wc *in_wc,
6343 const struct ib_grh *in_grh,
6344 const struct ib_mad_hdr *mad_hdr,
6345 size_t in_mad_size,
6346 struct ib_mad_hdr *out_mad,
6347 size_t *out_mad_size,
6348 u16 *out_mad_pkey_index)
6349
6350 #else
6351
6352 int
6353 qlnxr_process_mad(struct ib_device *ibdev,
6354 int process_mad_flags,
6355 u8 port_num,
6356 struct ib_wc *in_wc,
6357 struct ib_grh *in_grh,
6358 struct ib_mad *in_mad,
6359 struct ib_mad *out_mad)
6360
6361 #endif /* #if __FreeBSD_version >= 1102000 */
6362 {
6363 struct qlnxr_dev *dev;
6364 qlnx_host_t *ha;
6365
6366 dev = get_qlnxr_dev(ibdev);
6367 ha = dev->ha;
6368 QL_DPRINT12(ha, "process mad not supported\n");
6369
6370 return -ENOSYS;
6371 // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n",
6372 // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version,
6373 // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific,
6374 // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method,
6375 // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status);
6376
6377 // return IB_MAD_RESULT_SUCCESS;
6378 }
6379
6380 #if __FreeBSD_version >= 1102000
6381 int
6382 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num,
6383 struct ib_port_immutable *immutable)
6384 {
6385 struct qlnxr_dev *dev;
6386 qlnx_host_t *ha;
6387 struct ib_port_attr attr;
6388 int err;
6389
6390 dev = get_qlnxr_dev(ibdev);
6391 ha = dev->ha;
6392
6393 QL_DPRINT12(ha, "enter\n");
6394
6395 err = qlnxr_query_port(ibdev, port_num, &attr);
6396 if (err)
6397 return err;
6398
6399 if (QLNX_IS_IWARP(dev)) {
6400 immutable->pkey_tbl_len = 1;
6401 immutable->gid_tbl_len = 1;
6402 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
6403 immutable->max_mad_size = 0;
6404 } else {
6405 immutable->pkey_tbl_len = attr.pkey_tbl_len;
6406 immutable->gid_tbl_len = attr.gid_tbl_len;
6407 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
6408 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
6409 }
6410
6411 QL_DPRINT12(ha, "exit\n");
6412 return 0;
6413 }
6414 #endif /* #if __FreeBSD_version > 1102000 */
6415
6416 /***** iWARP related functions *************/
6417
6418 static void
6419 qlnxr_iw_mpa_request(void *context,
6420 struct ecore_iwarp_cm_event_params *params)
6421 {
6422 struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context;
6423 struct qlnxr_dev *dev = listener->dev;
6424 struct qlnxr_iw_ep *ep;
6425 struct iw_cm_event event;
6426 struct sockaddr_in *laddr;
6427 struct sockaddr_in *raddr;
6428 qlnx_host_t *ha;
6429
6430 ha = dev->ha;
6431
6432 QL_DPRINT12(ha, "enter\n");
6433
6434 if (params->cm_info->ip_version != ECORE_TCP_IPV4) {
6435 QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n",
6436 params->cm_info->ip_version);
6437 return;
6438 }
6439
6440 ep = kzalloc(sizeof(*ep), GFP_ATOMIC);
6441
6442 if (!ep) {
6443 QL_DPRINT11(ha, "kzalloc{ep) failed\n");
6444 return;
6445 }
6446
6447 ep->dev = dev;
6448 ep->ecore_context = params->ep_context;
6449
6450 memset(&event, 0, sizeof(event));
6451
6452 event.event = IW_CM_EVENT_CONNECT_REQUEST;
6453 event.status = params->status;
6454
6455 laddr = (struct sockaddr_in *)&event.local_addr;
6456 raddr = (struct sockaddr_in *)&event.remote_addr;
6457
6458 laddr->sin_family = AF_INET;
6459 raddr->sin_family = AF_INET;
6460
6461 laddr->sin_port = htons(params->cm_info->local_port);
6462 raddr->sin_port = htons(params->cm_info->remote_port);
6463
6464 laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]);
6465 raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]);
6466
6467 event.provider_data = (void *)ep;
6468 event.private_data = (void *)params->cm_info->private_data;
6469 event.private_data_len = (u8)params->cm_info->private_data_len;
6470
6471 #if __FreeBSD_version >= 1100000
6472 event.ord = params->cm_info->ord;
6473 event.ird = params->cm_info->ird;
6474 #endif /* #if __FreeBSD_version >= 1100000 */
6475
6476 listener->cm_id->event_handler(listener->cm_id, &event);
6477
6478 QL_DPRINT12(ha, "exit\n");
6479
6480 return;
6481 }
6482
6483 static void
6484 qlnxr_iw_issue_event(void *context,
6485 struct ecore_iwarp_cm_event_params *params,
6486 enum iw_cm_event_type event_type,
6487 char *str)
6488 {
6489 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6490 struct qlnxr_dev *dev = ep->dev;
6491 struct iw_cm_event event;
6492 qlnx_host_t *ha;
6493
6494 ha = dev->ha;
6495
6496 QL_DPRINT12(ha, "enter\n");
6497
6498 memset(&event, 0, sizeof(event));
6499 event.status = params->status;
6500 event.event = event_type;
6501
6502 if (params->cm_info != NULL) {
6503 #if __FreeBSD_version >= 1100000
6504 event.ird = params->cm_info->ird;
6505 event.ord = params->cm_info->ord;
6506 QL_DPRINT12(ha, "ord=[%d] \n", event.ord);
6507 QL_DPRINT12(ha, "ird=[%d] \n", event.ird);
6508 #endif /* #if __FreeBSD_version >= 1100000 */
6509
6510 event.private_data_len = params->cm_info->private_data_len;
6511 event.private_data = (void *)params->cm_info->private_data;
6512 QL_DPRINT12(ha, "private_data_len=[%d] \n",
6513 event.private_data_len);
6514 }
6515
6516 QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str);
6517 QL_DPRINT12(ha, "status=[%d] \n", event.status);
6518
6519 if (ep) {
6520 if (ep->cm_id)
6521 ep->cm_id->event_handler(ep->cm_id, &event);
6522 else
6523 QL_DPRINT11(ha, "ep->cm_id == NULL \n");
6524 } else {
6525 QL_DPRINT11(ha, "ep == NULL \n");
6526 }
6527
6528 QL_DPRINT12(ha, "exit\n");
6529
6530 return;
6531 }
6532
6533 static void
6534 qlnxr_iw_close_event(void *context,
6535 struct ecore_iwarp_cm_event_params *params)
6536 {
6537 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6538 struct qlnxr_dev *dev = ep->dev;
6539 qlnx_host_t *ha;
6540
6541 ha = dev->ha;
6542
6543 QL_DPRINT12(ha, "enter\n");
6544
6545 if (ep->cm_id) {
6546 qlnxr_iw_issue_event(context,
6547 params,
6548 IW_CM_EVENT_CLOSE,
6549 "IW_CM_EVENT_EVENT_CLOSE");
6550 ep->cm_id->rem_ref(ep->cm_id);
6551 ep->cm_id = NULL;
6552 }
6553
6554 QL_DPRINT12(ha, "exit\n");
6555
6556 return;
6557 }
6558
6559 #if __FreeBSD_version >= 1102000
6560
6561 static void
6562 qlnxr_iw_passive_complete(void *context,
6563 struct ecore_iwarp_cm_event_params *params)
6564 {
6565 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6566 struct qlnxr_dev *dev = ep->dev;
6567 qlnx_host_t *ha;
6568
6569 ha = dev->ha;
6570
6571 /* We will only reach the following state if MPA_REJECT was called on
6572 * passive. In this case there will be no associated QP.
6573 */
6574 if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) {
6575 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n");
6576 kfree(ep);
6577 return;
6578 }
6579
6580 /* We always issue an established event, however, ofed does not look
6581 * at event code for established. So if there was a failure, we follow
6582 * with close...
6583 */
6584 qlnxr_iw_issue_event(context,
6585 params,
6586 IW_CM_EVENT_ESTABLISHED,
6587 "IW_CM_EVENT_ESTABLISHED");
6588
6589 if (params->status < 0) {
6590 qlnxr_iw_close_event(context, params);
6591 }
6592
6593 return;
6594 }
6595
6596 struct qlnxr_discon_work {
6597 struct work_struct work;
6598 struct qlnxr_iw_ep *ep;
6599 enum ecore_iwarp_event_type event;
6600 int status;
6601 };
6602
6603 static void
6604 qlnxr_iw_disconnect_worker(struct work_struct *work)
6605 {
6606 struct qlnxr_discon_work *dwork =
6607 container_of(work, struct qlnxr_discon_work, work);
6608 struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
6609 struct qlnxr_iw_ep *ep = dwork->ep;
6610 struct qlnxr_dev *dev = ep->dev;
6611 struct qlnxr_qp *qp = ep->qp;
6612 struct iw_cm_event event;
6613
6614 if (qp->destroyed) {
6615 kfree(dwork);
6616 qlnxr_iw_qp_rem_ref(&qp->ibqp);
6617 return;
6618 }
6619
6620 memset(&event, 0, sizeof(event));
6621 event.status = dwork->status;
6622 event.event = IW_CM_EVENT_DISCONNECT;
6623
6624 /* Success means graceful disconnect was requested. modifying
6625 * to SQD is translated to graceful disconnect. O/w reset is sent
6626 */
6627 if (dwork->status)
6628 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR;
6629 else
6630 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD;
6631
6632 kfree(dwork);
6633
6634 if (ep->cm_id)
6635 ep->cm_id->event_handler(ep->cm_id, &event);
6636
6637 SET_FIELD(qp_params.modify_flags,
6638 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
6639
6640 ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
6641
6642 qlnxr_iw_qp_rem_ref(&qp->ibqp);
6643
6644 return;
6645 }
6646
6647 void
6648 qlnxr_iw_disconnect_event(void *context,
6649 struct ecore_iwarp_cm_event_params *params)
6650 {
6651 struct qlnxr_discon_work *work;
6652 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6653 struct qlnxr_dev *dev = ep->dev;
6654 struct qlnxr_qp *qp = ep->qp;
6655
6656 work = kzalloc(sizeof(*work), GFP_ATOMIC);
6657 if (!work)
6658 return;
6659
6660 qlnxr_iw_qp_add_ref(&qp->ibqp);
6661 work->ep = ep;
6662 work->event = params->event;
6663 work->status = params->status;
6664
6665 INIT_WORK(&work->work, qlnxr_iw_disconnect_worker);
6666 queue_work(dev->iwarp_wq, &work->work);
6667
6668 return;
6669 }
6670
6671 #endif /* #if __FreeBSD_version >= 1102000 */
6672
6673 static int
6674 qlnxr_iw_mpa_reply(void *context,
6675 struct ecore_iwarp_cm_event_params *params)
6676 {
6677 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6678 struct qlnxr_dev *dev = ep->dev;
6679 struct ecore_iwarp_send_rtr_in rtr_in;
6680 int rc;
6681 qlnx_host_t *ha;
6682
6683 ha = dev->ha;
6684
6685 QL_DPRINT12(ha, "enter\n");
6686
6687 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6688 return -EINVAL;
6689
6690 bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in));
6691 rtr_in.ep_context = params->ep_context;
6692
6693 rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in);
6694
6695 QL_DPRINT12(ha, "exit rc = %d\n", rc);
6696 return rc;
6697 }
6698
6699 void
6700 qlnxr_iw_qp_event(void *context,
6701 struct ecore_iwarp_cm_event_params *params,
6702 enum ib_event_type ib_event,
6703 char *str)
6704 {
6705 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6706 struct qlnxr_dev *dev = ep->dev;
6707 struct ib_qp *ibqp = &(ep->qp->ibqp);
6708 struct ib_event event;
6709 qlnx_host_t *ha;
6710
6711 ha = dev->ha;
6712
6713 QL_DPRINT12(ha,
6714 "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n",
6715 context, params->event, str, ibqp->event_handler);
6716
6717 if (ibqp->event_handler) {
6718 event.event = ib_event;
6719 event.device = ibqp->device;
6720 event.element.qp = ibqp;
6721 ibqp->event_handler(&event, ibqp->qp_context);
6722 }
6723
6724 return;
6725 }
6726
6727 int
6728 qlnxr_iw_event_handler(void *context,
6729 struct ecore_iwarp_cm_event_params *params)
6730 {
6731 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6732 struct qlnxr_dev *dev = ep->dev;
6733 qlnx_host_t *ha;
6734
6735 ha = dev->ha;
6736
6737 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6738 "enter\n", context, params->event);
6739
6740 switch (params->event) {
6741 /* Passive side request received */
6742 case ECORE_IWARP_EVENT_MPA_REQUEST:
6743 qlnxr_iw_mpa_request(context, params);
6744 break;
6745
6746 case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY:
6747 qlnxr_iw_mpa_reply(context, params);
6748 break;
6749
6750 /* Passive side established ( ack on mpa response ) */
6751 case ECORE_IWARP_EVENT_PASSIVE_COMPLETE:
6752
6753 #if __FreeBSD_version >= 1102000
6754
6755 ep->during_connect = 0;
6756 qlnxr_iw_passive_complete(context, params);
6757
6758 #else
6759 qlnxr_iw_issue_event(context,
6760 params,
6761 IW_CM_EVENT_ESTABLISHED,
6762 "IW_CM_EVENT_ESTABLISHED");
6763 #endif /* #if __FreeBSD_version >= 1102000 */
6764 break;
6765
6766 /* Active side reply received */
6767 case ECORE_IWARP_EVENT_ACTIVE_COMPLETE:
6768 ep->during_connect = 0;
6769 qlnxr_iw_issue_event(context,
6770 params,
6771 IW_CM_EVENT_CONNECT_REPLY,
6772 "IW_CM_EVENT_CONNECT_REPLY");
6773 if (params->status < 0) {
6774 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6775
6776 ep->cm_id->rem_ref(ep->cm_id);
6777 ep->cm_id = NULL;
6778 }
6779 break;
6780
6781 case ECORE_IWARP_EVENT_DISCONNECT:
6782
6783 #if __FreeBSD_version >= 1102000
6784 qlnxr_iw_disconnect_event(context, params);
6785 #else
6786 qlnxr_iw_issue_event(context,
6787 params,
6788 IW_CM_EVENT_DISCONNECT,
6789 "IW_CM_EVENT_DISCONNECT");
6790 qlnxr_iw_close_event(context, params);
6791 #endif /* #if __FreeBSD_version >= 1102000 */
6792 break;
6793
6794 case ECORE_IWARP_EVENT_CLOSE:
6795 ep->during_connect = 0;
6796 qlnxr_iw_close_event(context, params);
6797 break;
6798
6799 case ECORE_IWARP_EVENT_RQ_EMPTY:
6800 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6801 "IWARP_EVENT_RQ_EMPTY");
6802 break;
6803
6804 case ECORE_IWARP_EVENT_IRQ_FULL:
6805 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6806 "IWARP_EVENT_IRQ_FULL");
6807 break;
6808
6809 case ECORE_IWARP_EVENT_LLP_TIMEOUT:
6810 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6811 "IWARP_EVENT_LLP_TIMEOUT");
6812 break;
6813
6814 case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR:
6815 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6816 "IWARP_EVENT_REMOTE_PROTECTION_ERROR");
6817 break;
6818
6819 case ECORE_IWARP_EVENT_CQ_OVERFLOW:
6820 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6821 "QED_IWARP_EVENT_CQ_OVERFLOW");
6822 break;
6823
6824 case ECORE_IWARP_EVENT_QP_CATASTROPHIC:
6825 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6826 "QED_IWARP_EVENT_QP_CATASTROPHIC");
6827 break;
6828
6829 case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR:
6830 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6831 "IWARP_EVENT_LOCAL_ACCESS_ERROR");
6832 break;
6833
6834 case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR:
6835 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6836 "IWARP_EVENT_REMOTE_OPERATION_ERROR");
6837 break;
6838
6839 case ECORE_IWARP_EVENT_TERMINATE_RECEIVED:
6840 QL_DPRINT12(ha, "Got terminate message"
6841 " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n");
6842 break;
6843
6844 default:
6845 QL_DPRINT12(ha,
6846 "Unknown event [0x%x] received \n", params->event);
6847 break;
6848 };
6849
6850 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6851 "exit\n", context, params->event);
6852 return 0;
6853 }
6854
6855 static int
6856 qlnxr_addr4_resolve(struct qlnxr_dev *dev,
6857 struct sockaddr_in *src_in,
6858 struct sockaddr_in *dst_in,
6859 u8 *dst_mac)
6860 {
6861 int rc;
6862
6863 #if __FreeBSD_version >= 1100000
6864 rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in,
6865 dst_mac, NULL, NULL);
6866 #else
6867 struct llentry *lle;
6868
6869 rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in,
6870 dst_mac, &lle);
6871 #endif
6872
6873 QL_DPRINT12(dev->ha, "rc = %d "
6874 "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d "
6875 "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc,
6876 dst_in->sin_len, dst_in->sin_family,
6877 NIPQUAD((dst_in->sin_addr.s_addr)),
6878 dst_mac[0], dst_mac[1], dst_mac[2],
6879 dst_mac[3], dst_mac[4], dst_mac[5]);
6880
6881 return rc;
6882 }
6883
6884 int
6885 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
6886 {
6887 struct qlnxr_dev *dev;
6888 struct ecore_iwarp_connect_out out_params;
6889 struct ecore_iwarp_connect_in in_params;
6890 struct qlnxr_iw_ep *ep;
6891 struct qlnxr_qp *qp;
6892 struct sockaddr_in *laddr;
6893 struct sockaddr_in *raddr;
6894 int rc = 0;
6895 qlnx_host_t *ha;
6896
6897 dev = get_qlnxr_dev((cm_id->device));
6898 ha = dev->ha;
6899
6900 QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] "
6901 "enter \n", cm_id, conn_param);
6902
6903 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6904 return -EINVAL;
6905
6906 qp = idr_find(&dev->qpidr, conn_param->qpn);
6907
6908 laddr = (struct sockaddr_in *)&cm_id->local_addr;
6909 raddr = (struct sockaddr_in *)&cm_id->remote_addr;
6910
6911 QL_DPRINT12(ha,
6912 "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n",
6913 NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port,
6914 NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port);
6915
6916 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
6917 if (!ep) {
6918 QL_DPRINT11(ha, "struct qlnxr_iw_ep "
6919 "alloc memory failed\n");
6920 return -ENOMEM;
6921 }
6922
6923 ep->dev = dev;
6924 ep->qp = qp;
6925 cm_id->add_ref(cm_id);
6926 ep->cm_id = cm_id;
6927
6928 memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in));
6929 memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out));
6930
6931 in_params.event_cb = qlnxr_iw_event_handler;
6932 in_params.cb_context = ep;
6933
6934 in_params.cm_info.ip_version = ECORE_TCP_IPV4;
6935
6936 in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr);
6937 in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr);
6938 in_params.cm_info.remote_port = ntohs(raddr->sin_port);
6939 in_params.cm_info.local_port = ntohs(laddr->sin_port);
6940 in_params.cm_info.vlan = 0;
6941 in_params.mss = dev->ha->ifp->if_mtu - 40;
6942
6943 QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] "
6944 "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d "
6945 "vlan = %d\n",
6946 NIPQUAD((in_params.cm_info.remote_ip[0])),
6947 NIPQUAD((in_params.cm_info.local_ip[0])),
6948 in_params.cm_info.remote_port, in_params.cm_info.local_port,
6949 in_params.cm_info.vlan);
6950
6951 rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr);
6952
6953 if (rc) {
6954 QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n");
6955 goto err;
6956 }
6957
6958 QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p"
6959 " private_data_len=%d rq_psn=%d\n",
6960 conn_param->ord, conn_param->ird, conn_param->private_data,
6961 conn_param->private_data_len, qp->rq_psn);
6962
6963 in_params.cm_info.ord = conn_param->ord;
6964 in_params.cm_info.ird = conn_param->ird;
6965 in_params.cm_info.private_data = conn_param->private_data;
6966 in_params.cm_info.private_data_len = conn_param->private_data_len;
6967 in_params.qp = qp->ecore_qp;
6968
6969 memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
6970
6971 rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params);
6972
6973 if (rc) {
6974 QL_DPRINT12(ha, "ecore_iwarp_connect failed\n");
6975 goto err;
6976 }
6977
6978 QL_DPRINT12(ha, "exit\n");
6979
6980 return rc;
6981
6982 err:
6983 cm_id->rem_ref(cm_id);
6984 kfree(ep);
6985
6986 QL_DPRINT12(ha, "exit [%d]\n", rc);
6987 return rc;
6988 }
6989
6990 int
6991 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog)
6992 {
6993 struct qlnxr_dev *dev;
6994 struct qlnxr_iw_listener *listener;
6995 struct ecore_iwarp_listen_in iparams;
6996 struct ecore_iwarp_listen_out oparams;
6997 struct sockaddr_in *laddr;
6998 qlnx_host_t *ha;
6999 int rc;
7000
7001 dev = get_qlnxr_dev((cm_id->device));
7002 ha = dev->ha;
7003
7004 QL_DPRINT12(ha, "enter\n");
7005
7006 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7007 return -EINVAL;
7008
7009 laddr = (struct sockaddr_in *)&cm_id->local_addr;
7010
7011 listener = kzalloc(sizeof(*listener), GFP_KERNEL);
7012
7013 if (listener == NULL) {
7014 QL_DPRINT11(ha, "listener memory alloc failed\n");
7015 return -ENOMEM;
7016 }
7017
7018 listener->dev = dev;
7019 cm_id->add_ref(cm_id);
7020 listener->cm_id = cm_id;
7021 listener->backlog = backlog;
7022
7023 memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in));
7024 memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out));
7025
7026 iparams.cb_context = listener;
7027 iparams.event_cb = qlnxr_iw_event_handler;
7028 iparams.max_backlog = backlog;
7029
7030 iparams.ip_version = ECORE_TCP_IPV4;
7031
7032 iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr);
7033 iparams.port = ntohs(laddr->sin_port);
7034 iparams.vlan = 0;
7035
7036 QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n",
7037 NIPQUAD((laddr->sin_addr.s_addr)),
7038 laddr->sin_port, iparams.port);
7039
7040 rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams);
7041 if (rc) {
7042 QL_DPRINT11(ha,
7043 "ecore_iwarp_create_listen failed rc = %d\n", rc);
7044 goto err;
7045 }
7046
7047 listener->ecore_handle = oparams.handle;
7048 cm_id->provider_data = listener;
7049
7050 QL_DPRINT12(ha, "exit\n");
7051 return rc;
7052
7053 err:
7054 cm_id->rem_ref(cm_id);
7055 kfree(listener);
7056
7057 QL_DPRINT12(ha, "exit [%d]\n", rc);
7058 return rc;
7059 }
7060
7061 void
7062 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id)
7063 {
7064 struct qlnxr_iw_listener *listener = cm_id->provider_data;
7065 struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device));
7066 int rc = 0;
7067 qlnx_host_t *ha;
7068
7069 ha = dev->ha;
7070
7071 QL_DPRINT12(ha, "enter\n");
7072
7073 if (listener->ecore_handle)
7074 rc = ecore_iwarp_destroy_listen(dev->rdma_ctx,
7075 listener->ecore_handle);
7076
7077 cm_id->rem_ref(cm_id);
7078
7079 QL_DPRINT12(ha, "exit [%d]\n", rc);
7080 return;
7081 }
7082
7083 int
7084 qlnxr_iw_accept(struct iw_cm_id *cm_id,
7085 struct iw_cm_conn_param *conn_param)
7086 {
7087 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7088 struct qlnxr_dev *dev = ep->dev;
7089 struct qlnxr_qp *qp;
7090 struct ecore_iwarp_accept_in params;
7091 int rc;
7092 qlnx_host_t *ha;
7093
7094 ha = dev->ha;
7095
7096 QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn);
7097
7098 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7099 return -EINVAL;
7100
7101 qp = idr_find(&dev->qpidr, conn_param->qpn);
7102 if (!qp) {
7103 QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n",
7104 conn_param->qpn);
7105 return -EINVAL;
7106 }
7107 ep->qp = qp;
7108 qp->ep = ep;
7109 cm_id->add_ref(cm_id);
7110 ep->cm_id = cm_id;
7111
7112 params.ep_context = ep->ecore_context;
7113 params.cb_context = ep;
7114 params.qp = ep->qp->ecore_qp;
7115 params.private_data = conn_param->private_data;
7116 params.private_data_len = conn_param->private_data_len;
7117 params.ird = conn_param->ird;
7118 params.ord = conn_param->ord;
7119
7120 rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms);
7121 if (rc) {
7122 QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc);
7123 goto err;
7124 }
7125
7126 QL_DPRINT12(ha, "exit\n");
7127 return 0;
7128 err:
7129 cm_id->rem_ref(cm_id);
7130 QL_DPRINT12(ha, "exit rc = %d\n", rc);
7131 return rc;
7132 }
7133
7134 int
7135 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
7136 {
7137 #if __FreeBSD_version >= 1102000
7138
7139 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7140 struct qlnxr_dev *dev = ep->dev;
7141 struct ecore_iwarp_reject_in params;
7142 int rc;
7143
7144 params.ep_context = ep->ecore_context;
7145 params.cb_context = ep;
7146 params.private_data = pdata;
7147 params.private_data_len = pdata_len;
7148 ep->qp = NULL;
7149
7150 rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms);
7151
7152 return rc;
7153
7154 #else
7155
7156 printf("iWARP reject_cr not implemented\n");
7157 return -EINVAL;
7158
7159 #endif /* #if __FreeBSD_version >= 1102000 */
7160 }
7161
7162 void
7163 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp)
7164 {
7165 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7166 qlnx_host_t *ha;
7167
7168 ha = qp->dev->ha;
7169
7170 QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp);
7171
7172 atomic_inc(&qp->refcnt);
7173
7174 QL_DPRINT12(ha, "exit \n");
7175 return;
7176 }
7177
7178 void
7179 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp)
7180 {
7181 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7182 qlnx_host_t *ha;
7183
7184 ha = qp->dev->ha;
7185
7186 QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp);
7187
7188 if (atomic_dec_and_test(&qp->refcnt)) {
7189 qlnxr_idr_remove(qp->dev, qp->qp_id);
7190 kfree(qp);
7191 }
7192
7193 QL_DPRINT12(ha, "exit \n");
7194 return;
7195 }
7196
7197 struct ib_qp *
7198 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn)
7199 {
7200 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
7201 struct ib_qp *qp;
7202 qlnx_host_t *ha;
7203
7204 ha = dev->ha;
7205
7206 QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn);
7207
7208 qp = idr_find(&dev->qpidr, qpn);
7209
7210 QL_DPRINT12(ha, "exit qp = %p\n", qp);
7211
7212 return (qp);
7213 }
7214