1 /*
2 * Copyright (c) 2018-2019 Cavium, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
26 *
27 * $FreeBSD$
28 */
29
30 /*
31 * File: qlnxr_def.h
32 * Author: David C Somayajulu
33 */
34
35 #ifndef __QLNX_DEF_H_
36 #define __QLNX_DEF_H_
37
38 #include <sys/ktr.h>
39
40 #include <linux/list.h>
41 #include <linux/spinlock.h>
42 #include <linux/idr.h>
43 #include <linux/completion.h>
44 #include <linux/sched.h>
45 #include <linux/pci.h>
46 #include <linux/dma-mapping.h>
47 #include <linux/wait.h>
48 #include <linux/kref.h>
49 #include <linux/timer.h>
50 #include <linux/io.h>
51 #include <linux/fs.h>
52 #include <sys/vmem.h>
53
54 #include <asm/byteorder.h>
55
56 #include <netinet/in.h>
57 #include <net/ipv6.h>
58 #include <netinet/toecore.h>
59
60 #include <rdma/ib_smi.h>
61 #include <rdma/ib_user_verbs.h>
62 #include <rdma/ib_addr.h>
63 #include <rdma/ib_verbs.h>
64 #include <rdma/iw_cm.h>
65 #include <rdma/ib_umem.h>
66 #include <rdma/ib_mad.h>
67 #include <rdma/ib_sa.h>
68
69 #if __FreeBSD_version < 1100000
70 #undef MODULE_VERSION
71 #endif
72
73 #include "qlnx_os.h"
74 #include "bcm_osal.h"
75
76 #include "reg_addr.h"
77 #include "ecore_gtt_reg_addr.h"
78 #include "ecore.h"
79 #include "ecore_chain.h"
80 #include "ecore_status.h"
81 #include "ecore_hw.h"
82 #include "ecore_rt_defs.h"
83 #include "ecore_init_ops.h"
84 #include "ecore_int.h"
85 #include "ecore_cxt.h"
86 #include "ecore_spq.h"
87 #include "ecore_init_fw_funcs.h"
88 #include "ecore_sp_commands.h"
89 #include "ecore_dev_api.h"
90 #include "ecore_l2_api.h"
91 #ifdef CONFIG_ECORE_SRIOV
92 #include "ecore_sriov.h"
93 #include "ecore_vf.h"
94 #endif
95 #ifdef CONFIG_ECORE_LL2
96 #include "ecore_ll2.h"
97 #endif
98 #ifdef CONFIG_ECORE_FCOE
99 #include "ecore_fcoe.h"
100 #endif
101 #ifdef CONFIG_ECORE_ISCSI
102 #include "ecore_iscsi.h"
103 #endif
104 #include "ecore_mcp.h"
105 #include "ecore_hw_defs.h"
106 #include "mcp_public.h"
107
108 #ifdef CONFIG_ECORE_RDMA
109 #include "ecore_rdma.h"
110 #include "ecore_rdma_api.h"
111 #endif
112
113 #ifdef CONFIG_ECORE_ROCE
114 #include "ecore_roce.h"
115 #endif
116
117 #ifdef CONFIG_ECORE_IWARP
118 #include "ecore_iwarp.h"
119 #endif
120
121 #include "ecore_iro.h"
122 #include "nvm_cfg.h"
123
124 #include "ecore_dbg_fw_funcs.h"
125 #include "rdma_common.h"
126
127 #include "qlnx_ioctl.h"
128 #include "qlnx_def.h"
129 #include "qlnx_rdma.h"
130 #include "qlnxr_verbs.h"
131 #include "qlnxr_user.h"
132 #include "qlnx_ver.h"
133 #include <sys/smp.h>
134
135 #define QLNXR_ROCE_INTERFACE_VERSION 1801
136
137 #define QLNXR_MODULE_VERSION "8.18.1.0"
138 #define QLNXR_NODE_DESC "QLogic 579xx RoCE HCA"
139
140 #define OC_SKH_DEVICE_PF 0x720
141 #define OC_SKH_DEVICE_VF 0x728
142 #define QLNXR_MAX_AH 512
143
144 /* QLNXR Limitations */
145
146 /* SQ/RQ Limitations
147 * An S/RQ PBL contains a list a pointers to pages. Each page contains S/RQE
148 * elements. Several S/RQE elements make an S/RQE, up to a certain maximum that
149 * is different between SQ and RQ. The size of the PBL was chosen such as not to
150 * limit the MAX_WR supported by ECORE, and rounded up to a power of two.
151 */
152 /* SQ */
153 #define QLNXR_MAX_SQ_PBL (0x8000) /* 2^15 bytes */
154 #define QLNXR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *)) /* number */
155 #define QLNXR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge)) /* bytes */
156 #define QLNXR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \
157 QLNXR_SQE_ELEMENT_SIZE) /* number */
158 #define QLNXR_MAX_SQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
159 QLNXR_SQE_ELEMENT_SIZE) /* number */
160 #define QLNXR_MAX_SQE ((QLNXR_MAX_SQ_PBL_ENTRIES) * (RDMA_RING_PAGE_SIZE) / \
161 (QLNXR_SQE_ELEMENT_SIZE) / (QLNXR_MAX_SQE_ELEMENTS_PER_SQE))
162 /* RQ */
163 #define QLNXR_MAX_RQ_PBL (0x2000) /* 2^13 bytes */
164 #define QLNXR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *)) /* number */
165 #define QLNXR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge)) /* bytes */
166 #define QLNXR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE) /* number */
167 #define QLNXR_MAX_RQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
168 QLNXR_RQE_ELEMENT_SIZE) /* number */
169 #define QLNXR_MAX_RQE ((QLNXR_MAX_RQ_PBL_ENTRIES) * (RDMA_RING_PAGE_SIZE) / \
170 (QLNXR_RQE_ELEMENT_SIZE) / (QLNXR_MAX_RQE_ELEMENTS_PER_RQE))
171
172 /* CQE Limitation
173 * Although FW supports two layer PBL we use single layer since it is more
174 * than enough. For that layer we use a maximum size of 512 kB, again, because
175 * it reaches the maximum number of page pointers. Notice is the '-1' in the
176 * calculation that comes from having a u16 for the number of pages i.e. 0xffff
177 * is the maximum number of pages (in single layer).
178 */
179 #define QLNXR_CQE_SIZE (sizeof(union rdma_cqe))
180 #define QLNXR_MAX_CQE_PBL_SIZE (512*1024) /* 512kB */
181 #define QLNXR_MAX_CQE_PBL_ENTRIES (((QLNXR_MAX_CQE_PBL_SIZE) / \
182 sizeof(u64)) - 1) /* 64k -1 */
183 #define QLNXR_MAX_CQES ((u32)((QLNXR_MAX_CQE_PBL_ENTRIES) * (ECORE_CHAIN_PAGE_SIZE)\
184 / QLNXR_CQE_SIZE)) /* 8M -4096/32 = 8,388,480 */
185
186 /* CNQ size Limitation
187 * The maximum CNQ size is not reachable because the FW supports a chain of u16
188 * (specifically 64k-1). The FW can buffer CNQ elements avoiding an overflow, on
189 * the expense of performance. Hence we set it to an arbitrarily smaller value
190 * than the maximum.
191 */
192 #define QLNXR_ROCE_MAX_CNQ_SIZE (0x4000) /* 2^16 */
193
194 #define QLNXR_MAX_PORT (1)
195 #define QLNXR_PORT (1)
196
197 #define QLNXR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
198
199 #define convert_to_64bit(lo, hi) ((u64)hi << 32 | (u64)lo)
200
201 /* The following number is used to determine if a handle recevied from the FW
202 * actually point to a CQ/QP.
203 */
204 #define QLNXR_CQ_MAGIC_NUMBER (0x11223344)
205 #define QLNXR_QP_MAGIC_NUMBER (0x77889900)
206
207 /* Fast path debug prints */
208 #define FP_DP_VERBOSE(...)
209 /* #define FP_DP_VERBOSE(...) DP_VERBOSE(__VA_ARGS__) */
210
211 #define FW_PAGE_SIZE (RDMA_RING_PAGE_SIZE)
212
213 #define QLNXR_MSG_INIT 0x10000,
214 #define QLNXR_MSG_FAIL 0x10000,
215 #define QLNXR_MSG_CQ 0x20000,
216 #define QLNXR_MSG_RQ 0x40000,
217 #define QLNXR_MSG_SQ 0x80000,
218 #define QLNXR_MSG_QP (QLNXR_MSG_SQ | QLNXR_MSG_RQ),
219 #define QLNXR_MSG_MR 0x100000,
220 #define QLNXR_MSG_GSI 0x200000,
221 #define QLNXR_MSG_MISC 0x400000,
222 #define QLNXR_MSG_SRQ 0x800000,
223 #define QLNXR_MSG_IWARP 0x1000000,
224
225 #define QLNXR_ROCE_PKEY_MAX 1
226 #define QLNXR_ROCE_PKEY_TABLE_LEN 1
227 #define QLNXR_ROCE_PKEY_DEFAULT 0xffff
228
229 #define QLNXR_MAX_SGID 128 /* TBD - add more source gids... */
230
231 #define QLNXR_ENET_STATE_BIT (0)
232
233 #define QLNXR_MAX_MSIX (16)
234
235 struct qlnxr_cnq {
236 struct qlnxr_dev *dev;
237 struct ecore_chain pbl;
238 struct ecore_sb_info *sb;
239 char name[32];
240 u64 n_comp;
241 __le16 *hw_cons_ptr;
242 u8 index;
243 int irq_rid;
244 struct resource *irq;
245 void *irq_handle;
246 };
247
248 struct qlnxr_device_attr {
249 /* Vendor specific information */
250 u32 vendor_id;
251 u32 vendor_part_id;
252 u32 hw_ver;
253 u64 fw_ver;
254
255 u64 node_guid; /* node GUID */
256 u64 sys_image_guid; /* System image GUID */
257
258 u8 max_cnq;
259 u8 max_sge; /* Maximum # of scatter/gather entries
260 * per Work Request supported
261 */
262 u16 max_inline;
263 u32 max_sqe; /* Maximum number of send outstanding send work
264 * requests on any Work Queue supported
265 */
266 u32 max_rqe; /* Maximum number of receive outstanding receive
267 * work requests on any Work Queue supported
268 */
269 u8 max_qp_resp_rd_atomic_resc; /* Maximum number of RDMA Reads
270 * & atomic operation that can
271 * be outstanding per QP
272 */
273
274 u8 max_qp_req_rd_atomic_resc; /* The maximum depth per QP for
275 * initiation of RDMA Read
276 * & atomic operations
277 */
278 u64 max_dev_resp_rd_atomic_resc;
279 u32 max_cq;
280 u32 max_qp;
281 u32 max_mr; /* Maximum # of MRs supported */
282 u64 max_mr_size; /* Size (in bytes) of largest contiguous memory
283 * block that can be registered by this device
284 */
285 u32 max_cqe;
286 u32 max_mw; /* Maximum # of memory windows supported */
287 u32 max_fmr;
288 u32 max_mr_mw_fmr_pbl;
289 u64 max_mr_mw_fmr_size;
290 u32 max_pd; /* Maximum # of protection domains supported */
291 u32 max_ah;
292 u8 max_pkey;
293 u32 max_srq; /* Maximum number of SRQs */
294 u32 max_srq_wr; /* Maximum number of WRs per SRQ */
295 u8 max_srq_sge; /* Maximum number of SGE per WQE */
296 u8 max_stats_queues; /* Maximum number of statistics queues */
297 u32 dev_caps;
298
299 /* Abilty to support RNR-NAK generation */
300
301 #define QLNXR_ROCE_DEV_CAP_RNR_NAK_MASK 0x1
302 #define QLNXR_ROCE_DEV_CAP_RNR_NAK_SHIFT 0
303 /* Abilty to support shutdown port */
304 #define QLNXR_ROCE_DEV_CAP_SHUTDOWN_PORT_MASK 0x1
305 #define QLNXR_ROCE_DEV_CAP_SHUTDOWN_PORT_SHIFT 1
306 /* Abilty to support port active event */
307 #define QLNXR_ROCE_DEV_CAP_PORT_ACTIVE_EVENT_MASK 0x1
308 #define QLNXR_ROCE_DEV_CAP_PORT_ACTIVE_EVENT_SHIFT 2
309 /* Abilty to support port change event */
310 #define QLNXR_ROCE_DEV_CAP_PORT_CHANGE_EVENT_MASK 0x1
311 #define QLNXR_ROCE_DEV_CAP_PORT_CHANGE_EVENT_SHIFT 3
312 /* Abilty to support system image GUID */
313 #define QLNXR_ROCE_DEV_CAP_SYS_IMAGE_MASK 0x1
314 #define QLNXR_ROCE_DEV_CAP_SYS_IMAGE_SHIFT 4
315 /* Abilty to support bad P_Key counter support */
316 #define QLNXR_ROCE_DEV_CAP_BAD_PKEY_CNT_MASK 0x1
317 #define QLNXR_ROCE_DEV_CAP_BAD_PKEY_CNT_SHIFT 5
318 /* Abilty to support atomic operations */
319 #define QLNXR_ROCE_DEV_CAP_ATOMIC_OP_MASK 0x1
320 #define QLNXR_ROCE_DEV_CAP_ATOMIC_OP_SHIFT 6
321 #define QLNXR_ROCE_DEV_CAP_RESIZE_CQ_MASK 0x1
322 #define QLNXR_ROCE_DEV_CAP_RESIZE_CQ_SHIFT 7
323 /* Abilty to support modifying the maximum number of
324 * outstanding work requests per QP
325 */
326 #define QLNXR_ROCE_DEV_CAP_RESIZE_MAX_WR_MASK 0x1
327 #define QLNXR_ROCE_DEV_CAP_RESIZE_MAX_WR_SHIFT 8
328
329 /* Abilty to support automatic path migration */
330 #define QLNXR_ROCE_DEV_CAP_AUTO_PATH_MIG_MASK 0x1
331 #define QLNXR_ROCE_DEV_CAP_AUTO_PATH_MIG_SHIFT 9
332 /* Abilty to support the base memory management extensions */
333 #define QLNXR_ROCE_DEV_CAP_BASE_MEMORY_EXT_MASK 0x1
334 #define QLNXR_ROCE_DEV_CAP_BASE_MEMORY_EXT_SHIFT 10
335 #define QLNXR_ROCE_DEV_CAP_BASE_QUEUE_EXT_MASK 0x1
336 #define QLNXR_ROCE_DEV_CAP_BASE_QUEUE_EXT_SHIFT 11
337 /* Abilty to support multipile page sizes per memory region */
338 #define QLNXR_ROCE_DEV_CAP_MULTI_PAGE_PER_MR_EXT_MASK 0x1
339 #define QLNXR_ROCE_DEV_CAP_MULTI_PAGE_PER_MR_EXT_SHIFT 12
340 /* Abilty to support block list physical buffer list */
341 #define QLNXR_ROCE_DEV_CAP_BLOCK_MODE_MASK 0x1
342 #define QLNXR_ROCE_DEV_CAP_BLOCK_MODE_SHIFT 13
343 /* Abilty to support zero based virtual addresses */
344 #define QLNXR_ROCE_DEV_CAP_ZBVA_MASK 0x1
345 #define QLNXR_ROCE_DEV_CAP_ZBVA_SHIFT 14
346 /* Abilty to support local invalidate fencing */
347 #define QLNXR_ROCE_DEV_CAP_LOCAL_INV_FENCE_MASK 0x1
348 #define QLNXR_ROCE_DEV_CAP_LOCAL_INV_FENCE_SHIFT 15
349 /* Abilty to support Loopback on QP */
350 #define QLNXR_ROCE_DEV_CAP_LB_INDICATOR_MASK 0x1
351 #define QLNXR_ROCE_DEV_CAP_LB_INDICATOR_SHIFT 16
352 u64 page_size_caps;
353 u8 dev_ack_delay;
354 u32 reserved_lkey; /* Value of reserved L_key */
355 u32 bad_pkey_counter;/* Bad P_key counter support
356 * indicator
357 */
358 struct ecore_rdma_events events;
359 };
360
361 struct qlnxr_dev {
362 struct ib_device ibdev;
363 qlnx_host_t *ha;
364 struct ecore_dev *cdev;
365
366 /* Added to extend Applications Support */
367 struct pci_dev pdev;
368 uint32_t dp_module;
369 uint8_t dp_level;
370
371 void *rdma_ctx;
372
373 struct mtx idr_lock;
374 struct idr qpidr;
375
376 uint32_t wq_multiplier;
377 int num_cnq;
378
379 struct ecore_sb_info sb_array[QLNXR_MAX_MSIX];
380 struct qlnxr_cnq cnq_array[QLNXR_MAX_MSIX];
381
382 int sb_start;
383
384 int gsi_qp_created;
385 struct qlnxr_cq *gsi_sqcq;
386 struct qlnxr_cq *gsi_rqcq;
387 struct qlnxr_qp *gsi_qp;
388
389 /* TBD: we'll need an array of these probablly per DPI... */
390 void __iomem *db_addr;
391 uint64_t db_phys_addr;
392 uint32_t db_size;
393 uint16_t dpi;
394
395 uint64_t guid;
396 enum ib_atomic_cap atomic_cap;
397
398 union ib_gid sgid_tbl[QLNXR_MAX_SGID];
399 struct mtx sgid_lock;
400 struct notifier_block nb_inet;
401 struct notifier_block nb_inet6;
402
403 uint8_t mr_key;
404 struct list_head entry;
405
406 struct dentry *dbgfs;
407
408 uint8_t gsi_ll2_mac_address[ETH_ALEN];
409 uint8_t gsi_ll2_handle;
410
411 unsigned long enet_state;
412
413 struct workqueue_struct *iwarp_wq;
414
415 volatile uint32_t pd_count;
416 struct qlnxr_device_attr attr;
417 uint8_t user_dpm_enabled;
418 };
419
420 typedef struct qlnxr_dev qlnxr_dev_t;
421
422 struct qlnxr_pd {
423 struct ib_pd ibpd;
424 u32 pd_id;
425 struct qlnxr_ucontext *uctx;
426 };
427
428 struct qlnxr_ucontext {
429 struct ib_ucontext ibucontext;
430 struct qlnxr_dev *dev;
431 struct qlnxr_pd *pd;
432 u64 dpi_addr;
433 u64 dpi_phys_addr;
434 u32 dpi_size;
435 u16 dpi;
436
437 struct list_head mm_head;
438 struct mutex mm_list_lock;
439 };
440
441 struct qlnxr_dev_attr {
442 struct ib_device_attr ib_attr;
443 };
444
445 struct qlnxr_dma_mem {
446 void *va;
447 dma_addr_t pa;
448 u32 size;
449 };
450
451 struct qlnxr_pbl {
452 struct list_head list_entry;
453 void *va;
454 dma_addr_t pa;
455 };
456
457 struct qlnxr_queue_info {
458 void *va;
459 dma_addr_t dma;
460 u32 size;
461 u16 len;
462 u16 entry_size; /* Size of an element in the queue */
463 u16 id; /* qid, where to ring the doorbell. */
464 u16 head, tail;
465 bool created;
466 };
467
468 struct qlnxr_eq {
469 struct qlnxr_queue_info q;
470 u32 vector;
471 int cq_cnt;
472 struct qlnxr_dev *dev;
473 char irq_name[32];
474 };
475
476 struct qlnxr_mq {
477 struct qlnxr_queue_info sq;
478 struct qlnxr_queue_info cq;
479 bool rearm_cq;
480 };
481
482 struct phy_info {
483 u16 auto_speeds_supported;
484 u16 fixed_speeds_supported;
485 u16 phy_type;
486 u16 interface_type;
487 };
488
489 union db_prod64 {
490 struct rdma_pwm_val32_data data;
491 u64 raw;
492 };
493
494 enum qlnxr_cq_type {
495 QLNXR_CQ_TYPE_GSI,
496 QLNXR_CQ_TYPE_KERNEL,
497 QLNXR_CQ_TYPE_USER
498 };
499
500 struct qlnxr_pbl_info {
501 u32 num_pbls;
502 u32 num_pbes;
503 u32 pbl_size;
504 u32 pbe_size;
505 bool two_layered;
506 };
507
508 struct qlnxr_userq {
509 struct ib_umem *umem;
510 struct qlnxr_pbl_info pbl_info;
511 struct qlnxr_pbl *pbl_tbl;
512 u64 buf_addr;
513 size_t buf_len;
514 };
515
516 struct qlnxr_cq {
517 struct ib_cq ibcq; /* must be first */
518
519 enum qlnxr_cq_type cq_type;
520 uint32_t sig;
521 uint16_t icid;
522
523 /* relevant to cqs created from kernel space only (ULPs) */
524 spinlock_t cq_lock;
525 uint8_t arm_flags;
526 struct ecore_chain pbl;
527
528 void __iomem *db_addr; /* db address for cons update*/
529 union db_prod64 db;
530
531 uint8_t pbl_toggle;
532 union rdma_cqe *latest_cqe;
533 union rdma_cqe *toggle_cqe;
534
535 /* TODO: remove since it is redundant with 32 bit chains */
536 uint32_t cq_cons;
537
538 /* relevant to cqs created from user space only (applications) */
539 struct qlnxr_userq q;
540
541 /* destroy-IRQ handler race prevention */
542 uint8_t destroyed;
543 uint16_t cnq_notif;
544 };
545
546 struct qlnxr_ah {
547 struct ib_ah ibah;
548 struct ib_ah_attr attr;
549 };
550
551 union db_prod32 {
552 struct rdma_pwm_val16_data data;
553 u32 raw;
554 };
555
556 struct qlnxr_qp_hwq_info {
557 /* WQE Elements*/
558 struct ecore_chain pbl;
559 u64 p_phys_addr_tbl;
560 u32 max_sges;
561
562 /* WQE */
563 u16 prod; /* WQE prod index for SW ring */
564 u16 cons; /* WQE cons index for SW ring */
565 u16 wqe_cons;
566 u16 gsi_cons; /* filled in by GSI implementation */
567 u16 max_wr;
568
569 /* DB */
570 void __iomem *db; /* Doorbell address */
571 union db_prod32 db_data; /* Doorbell data */
572
573 /* Required for iwarp_only */
574 void __iomem *iwarp_db2; /* Doorbell address */
575 union db_prod32 iwarp_db2_data; /* Doorbell data */
576 };
577
578 #define QLNXR_INC_SW_IDX(p_info, index) \
579 do { \
580 p_info->index = (p_info->index + 1) & \
581 ecore_chain_get_capacity(p_info->pbl) \
582 } while (0)
583
584 struct qlnxr_srq_hwq_info {
585 u32 max_sges;
586 u32 max_wr;
587 struct ecore_chain pbl;
588 u64 p_phys_addr_tbl;
589 u32 wqe_prod; /* WQE prod index in HW ring */
590 u32 sge_prod; /* SGE prod index in HW ring */
591 u32 wr_prod_cnt; /* wr producer count */
592 u32 wr_cons_cnt; /* wr consumer count */
593 u32 num_elems;
594
595 u32 *virt_prod_pair_addr; /* producer pair virtual address */
596 dma_addr_t phy_prod_pair_addr; /* producer pair physical address */
597 };
598
599 struct qlnxr_srq {
600 struct ib_srq ibsrq;
601 struct qlnxr_dev *dev;
602 /* relevant to cqs created from user space only (applications) */
603 struct qlnxr_userq usrq;
604 struct qlnxr_srq_hwq_info hw_srq;
605 struct ib_umem *prod_umem;
606 u16 srq_id;
607 /* lock to protect srq recv post */
608 spinlock_t lock;
609 };
610
611 enum qlnxr_qp_err_bitmap {
612 QLNXR_QP_ERR_SQ_FULL = 1 << 0,
613 QLNXR_QP_ERR_RQ_FULL = 1 << 1,
614 QLNXR_QP_ERR_BAD_SR = 1 << 2,
615 QLNXR_QP_ERR_BAD_RR = 1 << 3,
616 QLNXR_QP_ERR_SQ_PBL_FULL = 1 << 4,
617 QLNXR_QP_ERR_RQ_PBL_FULL = 1 << 5,
618 };
619
620 struct mr_info {
621 struct qlnxr_pbl *pbl_table;
622 struct qlnxr_pbl_info pbl_info;
623 struct list_head free_pbl_list;
624 struct list_head inuse_pbl_list;
625 u32 completed;
626 u32 completed_handled;
627 };
628
629 #if __FreeBSD_version < 1102000
630 #define DEFINE_IB_FAST_REG
631 #else
632 #define DEFINE_ALLOC_MR
633 #endif
634
635 #ifdef DEFINE_IB_FAST_REG
636 struct qlnxr_fast_reg_page_list {
637 struct ib_fast_reg_page_list ibfrpl;
638 struct qlnxr_dev *dev;
639 struct mr_info info;
640 };
641 #endif
642 struct qlnxr_qp {
643 struct ib_qp ibqp; /* must be first */
644 struct qlnxr_dev *dev;
645 struct qlnxr_iw_ep *ep;
646 struct qlnxr_qp_hwq_info sq;
647 struct qlnxr_qp_hwq_info rq;
648
649 u32 max_inline_data;
650
651 #if __FreeBSD_version >= 1100000
652 spinlock_t q_lock ____cacheline_aligned;
653 #else
654 spinlock_t q_lock;
655 #endif
656
657 struct qlnxr_cq *sq_cq;
658 struct qlnxr_cq *rq_cq;
659 struct qlnxr_srq *srq;
660 enum ecore_roce_qp_state state; /* QP state */
661 u32 id;
662 struct qlnxr_pd *pd;
663 enum ib_qp_type qp_type;
664 struct ecore_rdma_qp *ecore_qp;
665 u32 qp_id;
666 u16 icid;
667 u16 mtu;
668 int sgid_idx;
669 u32 rq_psn;
670 u32 sq_psn;
671 u32 qkey;
672 u32 dest_qp_num;
673 u32 sig; /* unique siganture to identify valid QP */
674
675 /* relevant to qps created from kernel space only (ULPs) */
676 u8 prev_wqe_size;
677 u16 wqe_cons;
678 u32 err_bitmap;
679 bool signaled;
680 /* SQ shadow */
681 struct {
682 u64 wr_id;
683 enum ib_wc_opcode opcode;
684 u32 bytes_len;
685 u8 wqe_size;
686 bool signaled;
687 dma_addr_t icrc_mapping;
688 u32 *icrc;
689 #ifdef DEFINE_IB_FAST_REG
690 struct qlnxr_fast_reg_page_list *frmr;
691 #endif
692 struct qlnxr_mr *mr;
693 } *wqe_wr_id;
694
695 /* RQ shadow */
696 struct {
697 u64 wr_id;
698 struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE];
699 uint8_t wqe_size;
700
701 /* for GSI only */
702 u8 smac[ETH_ALEN];
703 u16 vlan_id;
704 int rc;
705 } *rqe_wr_id;
706
707 /* relevant to qps created from user space only (applications) */
708 struct qlnxr_userq usq;
709 struct qlnxr_userq urq;
710 atomic_t refcnt;
711 bool destroyed;
712 };
713
714 enum qlnxr_mr_type {
715 QLNXR_MR_USER,
716 QLNXR_MR_KERNEL,
717 QLNXR_MR_DMA,
718 QLNXR_MR_FRMR
719 };
720
721 struct qlnxr_mr {
722 struct ib_mr ibmr;
723 struct ib_umem *umem;
724
725 struct ecore_rdma_register_tid_in_params hw_mr;
726 enum qlnxr_mr_type type;
727
728 struct qlnxr_dev *dev;
729 struct mr_info info;
730
731 u64 *pages;
732 u32 npages;
733
734 u64 *iova_start; /* valid only for kernel_mr */
735 };
736
737 struct qlnxr_mm {
738 struct {
739 u64 phy_addr;
740 unsigned long len;
741 } key;
742 struct list_head entry;
743 };
744
745 struct qlnxr_iw_listener {
746 struct qlnxr_dev *dev;
747 struct iw_cm_id *cm_id;
748 int backlog;
749 void *ecore_handle;
750 };
751
752 struct qlnxr_iw_ep {
753 struct qlnxr_dev *dev;
754 struct iw_cm_id *cm_id;
755 struct qlnxr_qp *qp;
756 void *ecore_context;
757 u8 during_connect;
758 };
759
760 static inline void
qlnxr_inc_sw_cons(struct qlnxr_qp_hwq_info * info)761 qlnxr_inc_sw_cons(struct qlnxr_qp_hwq_info *info)
762 {
763 info->cons = (info->cons + 1) % info->max_wr;
764 info->wqe_cons++;
765 }
766
767 static inline void
qlnxr_inc_sw_prod(struct qlnxr_qp_hwq_info * info)768 qlnxr_inc_sw_prod(struct qlnxr_qp_hwq_info *info)
769 {
770 info->prod = (info->prod + 1) % info->max_wr;
771 }
772
773 static inline struct qlnxr_dev *
get_qlnxr_dev(struct ib_device * ibdev)774 get_qlnxr_dev(struct ib_device *ibdev)
775 {
776 return container_of(ibdev, struct qlnxr_dev, ibdev);
777 }
778
779 static inline struct qlnxr_ucontext *
get_qlnxr_ucontext(struct ib_ucontext * ibucontext)780 get_qlnxr_ucontext(struct ib_ucontext *ibucontext)
781 {
782 return container_of(ibucontext, struct qlnxr_ucontext, ibucontext);
783 }
784
785 static inline struct qlnxr_pd *
get_qlnxr_pd(struct ib_pd * ibpd)786 get_qlnxr_pd(struct ib_pd *ibpd)
787 {
788 return container_of(ibpd, struct qlnxr_pd, ibpd);
789 }
790
791 static inline struct qlnxr_cq *
get_qlnxr_cq(struct ib_cq * ibcq)792 get_qlnxr_cq(struct ib_cq *ibcq)
793 {
794 return container_of(ibcq, struct qlnxr_cq, ibcq);
795 }
796
797 static inline struct qlnxr_qp *
get_qlnxr_qp(struct ib_qp * ibqp)798 get_qlnxr_qp(struct ib_qp *ibqp)
799 {
800 return container_of(ibqp, struct qlnxr_qp, ibqp);
801 }
802
803 static inline struct qlnxr_mr *
get_qlnxr_mr(struct ib_mr * ibmr)804 get_qlnxr_mr(struct ib_mr *ibmr)
805 {
806 return container_of(ibmr, struct qlnxr_mr, ibmr);
807 }
808
809 static inline struct qlnxr_ah *
get_qlnxr_ah(struct ib_ah * ibah)810 get_qlnxr_ah(struct ib_ah *ibah)
811 {
812 return container_of(ibah, struct qlnxr_ah, ibah);
813 }
814
815 static inline struct qlnxr_srq *
get_qlnxr_srq(struct ib_srq * ibsrq)816 get_qlnxr_srq(struct ib_srq *ibsrq)
817 {
818 return container_of(ibsrq, struct qlnxr_srq, ibsrq);
819 }
820
qlnxr_qp_has_srq(struct qlnxr_qp * qp)821 static inline bool qlnxr_qp_has_srq(struct qlnxr_qp *qp)
822 {
823 return !!qp->srq;
824 }
825
qlnxr_qp_has_sq(struct qlnxr_qp * qp)826 static inline bool qlnxr_qp_has_sq(struct qlnxr_qp *qp)
827 {
828 if (qp->qp_type == IB_QPT_GSI)
829 return 0;
830
831 return 1;
832 }
833
qlnxr_qp_has_rq(struct qlnxr_qp * qp)834 static inline bool qlnxr_qp_has_rq(struct qlnxr_qp *qp)
835 {
836 if (qp->qp_type == IB_QPT_GSI || qlnxr_qp_has_srq(qp))
837 return 0;
838
839 return 1;
840 }
841
842 #ifdef DEFINE_IB_FAST_REG
get_qlnxr_frmr_list(struct ib_fast_reg_page_list * ifrpl)843 static inline struct qlnxr_fast_reg_page_list *get_qlnxr_frmr_list(
844 struct ib_fast_reg_page_list *ifrpl)
845 {
846 return container_of(ifrpl, struct qlnxr_fast_reg_page_list, ibfrpl);
847 }
848 #endif
849
850 #define SET_FIELD2(value, name, flag) \
851 do { \
852 (value) |= ((flag) << (name ## _SHIFT)); \
853 } while (0)
854
855 #define QLNXR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \
856 RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
857 #define QLNXR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \
858 RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
859 #define QLNXR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \
860 RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
861
862 #define QLNXR_RESP_RDMA_IMM (QLNXR_RESP_IMM | QLNXR_RESP_RDMA)
863
864 static inline int
qlnxr_get_dmac(struct qlnxr_dev * dev,struct ib_ah_attr * ah_attr,u8 * mac_addr)865 qlnxr_get_dmac(struct qlnxr_dev *dev, struct ib_ah_attr *ah_attr, u8 *mac_addr)
866 {
867 #ifdef DEFINE_NO_IP_BASED_GIDS
868 u8 *guid = &ah_attr->grh.dgid.raw[8]; /* GID's 64 MSBs are the GUID */
869 #endif
870 union ib_gid zero_sgid = { { 0 } };
871 struct in6_addr in6;
872
873 if (!memcmp(&ah_attr->grh.dgid, &zero_sgid, sizeof(union ib_gid))) {
874 memset(mac_addr, 0x00, ETH_ALEN);
875 return -EINVAL;
876 }
877
878 memcpy(&in6, ah_attr->grh.dgid.raw, sizeof(in6));
879
880 #ifdef DEFINE_NO_IP_BASED_GIDS
881 /* get the MAC address from the GUID i.e. EUI-64 to MAC address */
882 mac_addr[0] = guid[0] ^ 2; /* toggle the local/universal bit to local */
883 mac_addr[1] = guid[1];
884 mac_addr[2] = guid[2];
885 mac_addr[3] = guid[5];
886 mac_addr[4] = guid[6];
887 mac_addr[5] = guid[7];
888 #else
889 memcpy(mac_addr, ah_attr->dmac, ETH_ALEN);
890 #endif
891 return 0;
892 }
893
894 extern int qlnx_rdma_ll2_set_mac_filter(void *rdma_ctx, uint8_t *old_mac_address,
895 uint8_t *new_mac_address);
896
897 #define QLNXR_ROCE_PKEY_MAX 1
898 #define QLNXR_ROCE_PKEY_TABLE_LEN 1
899 #define QLNXR_ROCE_PKEY_DEFAULT 0xffff
900
901 #if __FreeBSD_version < 1100000
902 #define DEFINE_IB_AH_ATTR_WITH_DMAC (0)
903 #define DEFINE_IB_UMEM_WITH_CHUNK (1)
904 #else
905 #define DEFINE_IB_AH_ATTR_WITH_DMAC (1)
906 #endif
907
908 #define QLNX_IS_IWARP(rdev) IS_IWARP(ECORE_LEADING_HWFN(rdev->cdev))
909 #define QLNX_IS_ROCE(rdev) IS_ROCE(ECORE_LEADING_HWFN(rdev->cdev))
910
911 #define MAX_RXMIT_CONNS 16
912
913 #endif /* #ifndef __QLNX_DEF_H_ */
914