1 /* 2 * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the BSD-type 8 * license below: 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 17 * Redistributions in binary form must reproduce the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer in the documentation and/or other materials provided 20 * with the distribution. 21 * 22 * Neither the name of the Network Appliance, Inc. nor the names of 23 * its contributors may be used to endorse or promote products 24 * derived from this software without specific prior written 25 * permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 30 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 31 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 32 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 33 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 37 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 * 39 * Author: Tom Tucker <[email protected]> 40 */ 41 42 #ifndef SVC_RDMA_H 43 #define SVC_RDMA_H 44 #include <linux/sunrpc/xdr.h> 45 #include <linux/sunrpc/svcsock.h> 46 #include <linux/sunrpc/rpc_rdma.h> 47 #include <rdma/ib_verbs.h> 48 #include <rdma/rdma_cm.h> 49 #define SVCRDMA_DEBUG 50 51 /* RPC/RDMA parameters and stats */ 52 extern unsigned int svcrdma_ord; 53 extern unsigned int svcrdma_max_requests; 54 extern unsigned int svcrdma_max_bc_requests; 55 extern unsigned int svcrdma_max_req_size; 56 57 extern atomic_t rdma_stat_recv; 58 extern atomic_t rdma_stat_read; 59 extern atomic_t rdma_stat_write; 60 extern atomic_t rdma_stat_sq_starve; 61 extern atomic_t rdma_stat_rq_starve; 62 extern atomic_t rdma_stat_rq_poll; 63 extern atomic_t rdma_stat_rq_prod; 64 extern atomic_t rdma_stat_sq_poll; 65 extern atomic_t rdma_stat_sq_prod; 66 67 /* 68 * Contexts are built when an RDMA request is created and are a 69 * record of the resources that can be recovered when the request 70 * completes. 71 */ 72 struct svc_rdma_op_ctxt { 73 struct list_head free; 74 struct svc_rdma_op_ctxt *read_hdr; 75 struct svc_rdma_fastreg_mr *frmr; 76 int hdr_count; 77 struct xdr_buf arg; 78 struct ib_cqe cqe; 79 struct ib_cqe reg_cqe; 80 struct ib_cqe inv_cqe; 81 struct list_head dto_q; 82 enum ib_wc_status wc_status; 83 u32 byte_len; 84 u32 position; 85 struct svcxprt_rdma *xprt; 86 unsigned long flags; 87 enum dma_data_direction direction; 88 int count; 89 unsigned int mapped_sges; 90 struct ib_sge sge[RPCSVC_MAXPAGES]; 91 struct page *pages[RPCSVC_MAXPAGES]; 92 }; 93 94 /* 95 * NFS_ requests are mapped on the client side by the chunk lists in 96 * the RPCRDMA header. During the fetching of the RPC from the client 97 * and the writing of the reply to the client, the memory in the 98 * client and the memory in the server must be mapped as contiguous 99 * vaddr/len for access by the hardware. These data strucures keep 100 * these mappings. 101 * 102 * For an RDMA_WRITE, the 'sge' maps the RPC REPLY. For RDMA_READ, the 103 * 'sge' in the svc_rdma_req_map maps the server side RPC reply and the 104 * 'ch' field maps the read-list of the RPCRDMA header to the 'sge' 105 * mapping of the reply. 106 */ 107 struct svc_rdma_chunk_sge { 108 int start; /* sge no for this chunk */ 109 int count; /* sge count for this chunk */ 110 }; 111 struct svc_rdma_fastreg_mr { 112 struct ib_mr *mr; 113 struct scatterlist *sg; 114 int sg_nents; 115 unsigned long access_flags; 116 enum dma_data_direction direction; 117 struct list_head frmr_list; 118 }; 119 struct svc_rdma_req_map { 120 struct list_head free; 121 unsigned long count; 122 union { 123 struct kvec sge[RPCSVC_MAXPAGES]; 124 struct svc_rdma_chunk_sge ch[RPCSVC_MAXPAGES]; 125 unsigned long lkey[RPCSVC_MAXPAGES]; 126 }; 127 }; 128 #define RDMACTXT_F_LAST_CTXT 2 129 130 #define SVCRDMA_DEVCAP_FAST_REG 1 /* fast mr registration */ 131 #define SVCRDMA_DEVCAP_READ_W_INV 2 /* read w/ invalidate */ 132 133 struct svcxprt_rdma { 134 struct svc_xprt sc_xprt; /* SVC transport structure */ 135 struct rdma_cm_id *sc_cm_id; /* RDMA connection id */ 136 struct list_head sc_accept_q; /* Conn. waiting accept */ 137 int sc_ord; /* RDMA read limit */ 138 int sc_max_sge; 139 int sc_max_sge_rd; /* max sge for read target */ 140 bool sc_snd_w_inv; /* OK to use Send With Invalidate */ 141 142 atomic_t sc_sq_count; /* Number of SQ WR on queue */ 143 unsigned int sc_sq_depth; /* Depth of SQ */ 144 unsigned int sc_rq_depth; /* Depth of RQ */ 145 u32 sc_max_requests; /* Forward credits */ 146 u32 sc_max_bc_requests;/* Backward credits */ 147 int sc_max_req_size; /* Size of each RQ WR buf */ 148 149 struct ib_pd *sc_pd; 150 151 atomic_t sc_dma_used; 152 spinlock_t sc_ctxt_lock; 153 struct list_head sc_ctxts; 154 int sc_ctxt_used; 155 spinlock_t sc_map_lock; 156 struct list_head sc_maps; 157 158 struct list_head sc_rq_dto_q; 159 spinlock_t sc_rq_dto_lock; 160 struct ib_qp *sc_qp; 161 struct ib_cq *sc_rq_cq; 162 struct ib_cq *sc_sq_cq; 163 int (*sc_reader)(struct svcxprt_rdma *, 164 struct svc_rqst *, 165 struct svc_rdma_op_ctxt *, 166 int *, u32 *, u32, u32, u64, bool); 167 u32 sc_dev_caps; /* distilled device caps */ 168 unsigned int sc_frmr_pg_list_len; 169 struct list_head sc_frmr_q; 170 spinlock_t sc_frmr_q_lock; 171 172 spinlock_t sc_lock; /* transport lock */ 173 174 wait_queue_head_t sc_send_wait; /* SQ exhaustion waitlist */ 175 unsigned long sc_flags; 176 struct list_head sc_dto_q; /* DTO tasklet I/O pending Q */ 177 struct list_head sc_read_complete_q; 178 struct work_struct sc_work; 179 }; 180 /* sc_flags */ 181 #define RDMAXPRT_CONN_PENDING 3 182 183 #define RPCRDMA_LISTEN_BACKLOG 10 184 /* The default ORD value is based on two outstanding full-size writes with a 185 * page size of 4k, or 32k * 2 ops / 4k = 16 outstanding RDMA_READ. */ 186 #define RPCRDMA_ORD (64/4) 187 #define RPCRDMA_SQ_DEPTH_MULT 8 188 #define RPCRDMA_MAX_REQUESTS 32 189 #define RPCRDMA_MAX_REQ_SIZE 4096 190 191 /* Typical ULP usage of BC requests is NFSv4.1 backchannel. Our 192 * current NFSv4.1 implementation supports one backchannel slot. 193 */ 194 #define RPCRDMA_MAX_BC_REQUESTS 2 195 196 #define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD 197 198 /* Track DMA maps for this transport and context */ 199 static inline void svc_rdma_count_mappings(struct svcxprt_rdma *rdma, 200 struct svc_rdma_op_ctxt *ctxt) 201 { 202 ctxt->mapped_sges++; 203 atomic_inc(&rdma->sc_dma_used); 204 } 205 206 /* svc_rdma_backchannel.c */ 207 extern int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, 208 struct rpcrdma_msg *rmsgp, 209 struct xdr_buf *rcvbuf); 210 211 /* svc_rdma_marshal.c */ 212 extern int svc_rdma_xdr_decode_req(struct xdr_buf *); 213 extern int svc_rdma_xdr_encode_error(struct svcxprt_rdma *, 214 struct rpcrdma_msg *, 215 enum rpcrdma_errcode, __be32 *); 216 extern void svc_rdma_xdr_encode_write_list(struct rpcrdma_msg *, int); 217 extern void svc_rdma_xdr_encode_reply_array(struct rpcrdma_write_array *, int); 218 extern void svc_rdma_xdr_encode_array_chunk(struct rpcrdma_write_array *, int, 219 __be32, __be64, u32); 220 extern void svc_rdma_xdr_encode_reply_header(struct svcxprt_rdma *, 221 struct rpcrdma_msg *, 222 struct rpcrdma_msg *, 223 enum rpcrdma_proc); 224 extern int svc_rdma_xdr_get_reply_hdr_len(struct rpcrdma_msg *); 225 226 /* svc_rdma_recvfrom.c */ 227 extern int svc_rdma_recvfrom(struct svc_rqst *); 228 extern int rdma_read_chunk_lcl(struct svcxprt_rdma *, struct svc_rqst *, 229 struct svc_rdma_op_ctxt *, int *, u32 *, 230 u32, u32, u64, bool); 231 extern int rdma_read_chunk_frmr(struct svcxprt_rdma *, struct svc_rqst *, 232 struct svc_rdma_op_ctxt *, int *, u32 *, 233 u32, u32, u64, bool); 234 235 /* svc_rdma_sendto.c */ 236 extern int svc_rdma_map_xdr(struct svcxprt_rdma *, struct xdr_buf *, 237 struct svc_rdma_req_map *, bool); 238 extern int svc_rdma_sendto(struct svc_rqst *); 239 extern struct rpcrdma_read_chunk * 240 svc_rdma_get_read_chunk(struct rpcrdma_msg *); 241 extern void svc_rdma_send_error(struct svcxprt_rdma *, struct rpcrdma_msg *, 242 int); 243 244 /* svc_rdma_transport.c */ 245 extern void svc_rdma_wc_send(struct ib_cq *, struct ib_wc *); 246 extern void svc_rdma_wc_write(struct ib_cq *, struct ib_wc *); 247 extern void svc_rdma_wc_reg(struct ib_cq *, struct ib_wc *); 248 extern void svc_rdma_wc_read(struct ib_cq *, struct ib_wc *); 249 extern void svc_rdma_wc_inv(struct ib_cq *, struct ib_wc *); 250 extern int svc_rdma_send(struct svcxprt_rdma *, struct ib_send_wr *); 251 extern int svc_rdma_post_recv(struct svcxprt_rdma *, gfp_t); 252 extern int svc_rdma_repost_recv(struct svcxprt_rdma *, gfp_t); 253 extern int svc_rdma_create_listen(struct svc_serv *, int, struct sockaddr *); 254 extern struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *); 255 extern void svc_rdma_put_context(struct svc_rdma_op_ctxt *, int); 256 extern void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt *ctxt); 257 extern struct svc_rdma_req_map *svc_rdma_get_req_map(struct svcxprt_rdma *); 258 extern void svc_rdma_put_req_map(struct svcxprt_rdma *, 259 struct svc_rdma_req_map *); 260 extern struct svc_rdma_fastreg_mr *svc_rdma_get_frmr(struct svcxprt_rdma *); 261 extern void svc_rdma_put_frmr(struct svcxprt_rdma *, 262 struct svc_rdma_fastreg_mr *); 263 extern void svc_sq_reap(struct svcxprt_rdma *); 264 extern void svc_rq_reap(struct svcxprt_rdma *); 265 extern void svc_rdma_prep_reply_hdr(struct svc_rqst *); 266 267 extern struct svc_xprt_class svc_rdma_class; 268 #ifdef CONFIG_SUNRPC_BACKCHANNEL 269 extern struct svc_xprt_class svc_rdma_bc_class; 270 #endif 271 272 /* svc_rdma.c */ 273 extern struct workqueue_struct *svc_rdma_wq; 274 extern int svc_rdma_init(void); 275 extern void svc_rdma_cleanup(void); 276 277 #endif 278