1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
3 *
4 * Copyright (c) 2005 Topspin Communications. All rights reserved.
5 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
6 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
7 * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
8 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
9 *
10 * This software is available to you under a choice of one of two
11 * licenses. You may choose to be licensed under the terms of the GNU
12 * General Public License (GPL) Version 2, available from the file
13 * COPYING in the main directory of this source tree, or the
14 * OpenIB.org BSD license below:
15 *
16 * Redistribution and use in source and binary forms, with or
17 * without modification, are permitted provided that the following
18 * conditions are met:
19 *
20 * - Redistributions of source code must retain the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer.
23 *
24 * - Redistributions in binary form must reproduce the above
25 * copyright notice, this list of conditions and the following
26 * disclaimer in the documentation and/or other materials
27 * provided with the distribution.
28 *
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 * SOFTWARE.
37 */
38
39 #include <sys/cdefs.h>
40 __FBSDID("$FreeBSD$");
41
42 #include <linux/module.h>
43 #include <linux/device.h>
44 #include <linux/err.h>
45 #include <linux/fs.h>
46 #include <linux/poll.h>
47 #include <linux/sched.h>
48 #include <linux/file.h>
49 #include <linux/cdev.h>
50 #include <linux/slab.h>
51 #include <linux/pci.h>
52
53 #include <asm/uaccess.h>
54
55 #include <rdma/ib.h>
56
57 #include "uverbs.h"
58
59 MODULE_AUTHOR("Roland Dreier");
60 MODULE_DESCRIPTION("InfiniBand userspace verbs access");
61 MODULE_LICENSE("Dual BSD/GPL");
62
63 enum {
64 IB_UVERBS_MAJOR = 231,
65 IB_UVERBS_BASE_MINOR = 192,
66 IB_UVERBS_MAX_DEVICES = 32
67 };
68
69 #define IB_UVERBS_BASE_DEV MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR)
70
71 static struct class *uverbs_class;
72
73 DEFINE_SPINLOCK(ib_uverbs_idr_lock);
74 DEFINE_IDR(ib_uverbs_pd_idr);
75 DEFINE_IDR(ib_uverbs_mr_idr);
76 DEFINE_IDR(ib_uverbs_mw_idr);
77 DEFINE_IDR(ib_uverbs_ah_idr);
78 DEFINE_IDR(ib_uverbs_cq_idr);
79 DEFINE_IDR(ib_uverbs_qp_idr);
80 DEFINE_IDR(ib_uverbs_srq_idr);
81 DEFINE_IDR(ib_uverbs_xrcd_idr);
82 DEFINE_IDR(ib_uverbs_rule_idr);
83 DEFINE_IDR(ib_uverbs_wq_idr);
84 DEFINE_IDR(ib_uverbs_rwq_ind_tbl_idr);
85
86 static DEFINE_SPINLOCK(map_lock);
87 static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES);
88
89 static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file,
90 struct ib_device *ib_dev,
91 const char __user *buf, int in_len,
92 int out_len) = {
93 [IB_USER_VERBS_CMD_GET_CONTEXT] = ib_uverbs_get_context,
94 [IB_USER_VERBS_CMD_QUERY_DEVICE] = ib_uverbs_query_device,
95 [IB_USER_VERBS_CMD_QUERY_PORT] = ib_uverbs_query_port,
96 [IB_USER_VERBS_CMD_ALLOC_PD] = ib_uverbs_alloc_pd,
97 [IB_USER_VERBS_CMD_DEALLOC_PD] = ib_uverbs_dealloc_pd,
98 [IB_USER_VERBS_CMD_REG_MR] = ib_uverbs_reg_mr,
99 [IB_USER_VERBS_CMD_REREG_MR] = ib_uverbs_rereg_mr,
100 [IB_USER_VERBS_CMD_DEREG_MR] = ib_uverbs_dereg_mr,
101 [IB_USER_VERBS_CMD_ALLOC_MW] = ib_uverbs_alloc_mw,
102 [IB_USER_VERBS_CMD_DEALLOC_MW] = ib_uverbs_dealloc_mw,
103 [IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL] = ib_uverbs_create_comp_channel,
104 [IB_USER_VERBS_CMD_CREATE_CQ] = ib_uverbs_create_cq,
105 [IB_USER_VERBS_CMD_RESIZE_CQ] = ib_uverbs_resize_cq,
106 [IB_USER_VERBS_CMD_POLL_CQ] = ib_uverbs_poll_cq,
107 [IB_USER_VERBS_CMD_REQ_NOTIFY_CQ] = ib_uverbs_req_notify_cq,
108 [IB_USER_VERBS_CMD_DESTROY_CQ] = ib_uverbs_destroy_cq,
109 [IB_USER_VERBS_CMD_CREATE_QP] = ib_uverbs_create_qp,
110 [IB_USER_VERBS_CMD_QUERY_QP] = ib_uverbs_query_qp,
111 [IB_USER_VERBS_CMD_MODIFY_QP] = ib_uverbs_modify_qp,
112 [IB_USER_VERBS_CMD_DESTROY_QP] = ib_uverbs_destroy_qp,
113 [IB_USER_VERBS_CMD_POST_SEND] = ib_uverbs_post_send,
114 [IB_USER_VERBS_CMD_POST_RECV] = ib_uverbs_post_recv,
115 [IB_USER_VERBS_CMD_POST_SRQ_RECV] = ib_uverbs_post_srq_recv,
116 [IB_USER_VERBS_CMD_CREATE_AH] = ib_uverbs_create_ah,
117 [IB_USER_VERBS_CMD_DESTROY_AH] = ib_uverbs_destroy_ah,
118 [IB_USER_VERBS_CMD_ATTACH_MCAST] = ib_uverbs_attach_mcast,
119 [IB_USER_VERBS_CMD_DETACH_MCAST] = ib_uverbs_detach_mcast,
120 [IB_USER_VERBS_CMD_CREATE_SRQ] = ib_uverbs_create_srq,
121 [IB_USER_VERBS_CMD_MODIFY_SRQ] = ib_uverbs_modify_srq,
122 [IB_USER_VERBS_CMD_QUERY_SRQ] = ib_uverbs_query_srq,
123 [IB_USER_VERBS_CMD_DESTROY_SRQ] = ib_uverbs_destroy_srq,
124 [IB_USER_VERBS_CMD_OPEN_XRCD] = ib_uverbs_open_xrcd,
125 [IB_USER_VERBS_CMD_CLOSE_XRCD] = ib_uverbs_close_xrcd,
126 [IB_USER_VERBS_CMD_CREATE_XSRQ] = ib_uverbs_create_xsrq,
127 [IB_USER_VERBS_CMD_OPEN_QP] = ib_uverbs_open_qp,
128 };
129
130 static int (*uverbs_ex_cmd_table[])(struct ib_uverbs_file *file,
131 struct ib_device *ib_dev,
132 struct ib_udata *ucore,
133 struct ib_udata *uhw) = {
134 [IB_USER_VERBS_EX_CMD_CREATE_FLOW] = ib_uverbs_ex_create_flow,
135 [IB_USER_VERBS_EX_CMD_DESTROY_FLOW] = ib_uverbs_ex_destroy_flow,
136 [IB_USER_VERBS_EX_CMD_QUERY_DEVICE] = ib_uverbs_ex_query_device,
137 [IB_USER_VERBS_EX_CMD_CREATE_CQ] = ib_uverbs_ex_create_cq,
138 [IB_USER_VERBS_EX_CMD_CREATE_QP] = ib_uverbs_ex_create_qp,
139 [IB_USER_VERBS_EX_CMD_CREATE_WQ] = ib_uverbs_ex_create_wq,
140 [IB_USER_VERBS_EX_CMD_MODIFY_WQ] = ib_uverbs_ex_modify_wq,
141 [IB_USER_VERBS_EX_CMD_DESTROY_WQ] = ib_uverbs_ex_destroy_wq,
142 [IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL] = ib_uverbs_ex_create_rwq_ind_table,
143 [IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL] = ib_uverbs_ex_destroy_rwq_ind_table,
144 };
145
146 static void ib_uverbs_add_one(struct ib_device *device);
147 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data);
148
uverbs_dealloc_mw(struct ib_mw * mw)149 int uverbs_dealloc_mw(struct ib_mw *mw)
150 {
151 struct ib_pd *pd = mw->pd;
152 int ret;
153
154 ret = mw->device->dealloc_mw(mw);
155 if (!ret)
156 atomic_dec(&pd->usecnt);
157 return ret;
158 }
159
ib_uverbs_release_dev(struct kobject * kobj)160 static void ib_uverbs_release_dev(struct kobject *kobj)
161 {
162 struct ib_uverbs_device *dev =
163 container_of(kobj, struct ib_uverbs_device, kobj);
164
165 cleanup_srcu_struct(&dev->disassociate_srcu);
166 kfree(dev);
167 }
168
169 static struct kobj_type ib_uverbs_dev_ktype = {
170 .release = ib_uverbs_release_dev,
171 };
172
ib_uverbs_release_event_file(struct kref * ref)173 static void ib_uverbs_release_event_file(struct kref *ref)
174 {
175 struct ib_uverbs_event_file *file =
176 container_of(ref, struct ib_uverbs_event_file, ref);
177
178 kfree(file);
179 }
180
ib_uverbs_release_ucq(struct ib_uverbs_file * file,struct ib_uverbs_event_file * ev_file,struct ib_ucq_object * uobj)181 void ib_uverbs_release_ucq(struct ib_uverbs_file *file,
182 struct ib_uverbs_event_file *ev_file,
183 struct ib_ucq_object *uobj)
184 {
185 struct ib_uverbs_event *evt, *tmp;
186
187 if (ev_file) {
188 spin_lock_irq(&ev_file->lock);
189 list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) {
190 list_del(&evt->list);
191 kfree(evt);
192 }
193 spin_unlock_irq(&ev_file->lock);
194
195 kref_put(&ev_file->ref, ib_uverbs_release_event_file);
196 }
197
198 spin_lock_irq(&file->async_file->lock);
199 list_for_each_entry_safe(evt, tmp, &uobj->async_list, obj_list) {
200 list_del(&evt->list);
201 kfree(evt);
202 }
203 spin_unlock_irq(&file->async_file->lock);
204 }
205
ib_uverbs_release_uevent(struct ib_uverbs_file * file,struct ib_uevent_object * uobj)206 void ib_uverbs_release_uevent(struct ib_uverbs_file *file,
207 struct ib_uevent_object *uobj)
208 {
209 struct ib_uverbs_event *evt, *tmp;
210
211 spin_lock_irq(&file->async_file->lock);
212 list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) {
213 list_del(&evt->list);
214 kfree(evt);
215 }
216 spin_unlock_irq(&file->async_file->lock);
217 }
218
ib_uverbs_detach_umcast(struct ib_qp * qp,struct ib_uqp_object * uobj)219 static void ib_uverbs_detach_umcast(struct ib_qp *qp,
220 struct ib_uqp_object *uobj)
221 {
222 struct ib_uverbs_mcast_entry *mcast, *tmp;
223
224 list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) {
225 ib_detach_mcast(qp, &mcast->gid, mcast->lid);
226 list_del(&mcast->list);
227 kfree(mcast);
228 }
229 }
230
ib_uverbs_cleanup_ucontext(struct ib_uverbs_file * file,struct ib_ucontext * context)231 static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file,
232 struct ib_ucontext *context)
233 {
234 struct ib_uobject *uobj, *tmp;
235
236 context->closing = 1;
237
238 list_for_each_entry_safe(uobj, tmp, &context->ah_list, list) {
239 struct ib_ah *ah = uobj->object;
240
241 idr_remove_uobj(&ib_uverbs_ah_idr, uobj);
242 ib_destroy_ah(ah);
243 kfree(uobj);
244 }
245
246 /* Remove MWs before QPs, in order to support type 2A MWs. */
247 list_for_each_entry_safe(uobj, tmp, &context->mw_list, list) {
248 struct ib_mw *mw = uobj->object;
249
250 idr_remove_uobj(&ib_uverbs_mw_idr, uobj);
251 uverbs_dealloc_mw(mw);
252 kfree(uobj);
253 }
254
255 list_for_each_entry_safe(uobj, tmp, &context->rule_list, list) {
256 struct ib_flow *flow_id = uobj->object;
257
258 idr_remove_uobj(&ib_uverbs_rule_idr, uobj);
259 ib_destroy_flow(flow_id);
260 kfree(uobj);
261 }
262
263 list_for_each_entry_safe(uobj, tmp, &context->qp_list, list) {
264 struct ib_qp *qp = uobj->object;
265 struct ib_uqp_object *uqp =
266 container_of(uobj, struct ib_uqp_object, uevent.uobject);
267
268 idr_remove_uobj(&ib_uverbs_qp_idr, uobj);
269 if (qp == qp->real_qp)
270 ib_uverbs_detach_umcast(qp, uqp);
271 ib_destroy_qp(qp);
272 ib_uverbs_release_uevent(file, &uqp->uevent);
273 kfree(uqp);
274 }
275
276 list_for_each_entry_safe(uobj, tmp, &context->rwq_ind_tbl_list, list) {
277 struct ib_rwq_ind_table *rwq_ind_tbl = uobj->object;
278 struct ib_wq **ind_tbl = rwq_ind_tbl->ind_tbl;
279
280 idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj);
281 ib_destroy_rwq_ind_table(rwq_ind_tbl);
282 kfree(ind_tbl);
283 kfree(uobj);
284 }
285
286 list_for_each_entry_safe(uobj, tmp, &context->wq_list, list) {
287 struct ib_wq *wq = uobj->object;
288 struct ib_uwq_object *uwq =
289 container_of(uobj, struct ib_uwq_object, uevent.uobject);
290
291 idr_remove_uobj(&ib_uverbs_wq_idr, uobj);
292 ib_destroy_wq(wq);
293 ib_uverbs_release_uevent(file, &uwq->uevent);
294 kfree(uwq);
295 }
296
297 list_for_each_entry_safe(uobj, tmp, &context->srq_list, list) {
298 struct ib_srq *srq = uobj->object;
299 struct ib_uevent_object *uevent =
300 container_of(uobj, struct ib_uevent_object, uobject);
301
302 idr_remove_uobj(&ib_uverbs_srq_idr, uobj);
303 ib_destroy_srq(srq);
304 ib_uverbs_release_uevent(file, uevent);
305 kfree(uevent);
306 }
307
308 list_for_each_entry_safe(uobj, tmp, &context->cq_list, list) {
309 struct ib_cq *cq = uobj->object;
310 struct ib_uverbs_event_file *ev_file = cq->cq_context;
311 struct ib_ucq_object *ucq =
312 container_of(uobj, struct ib_ucq_object, uobject);
313
314 idr_remove_uobj(&ib_uverbs_cq_idr, uobj);
315 ib_destroy_cq(cq);
316 ib_uverbs_release_ucq(file, ev_file, ucq);
317 kfree(ucq);
318 }
319
320 list_for_each_entry_safe(uobj, tmp, &context->mr_list, list) {
321 struct ib_mr *mr = uobj->object;
322
323 idr_remove_uobj(&ib_uverbs_mr_idr, uobj);
324 ib_dereg_mr(mr);
325 kfree(uobj);
326 }
327
328 mutex_lock(&file->device->xrcd_tree_mutex);
329 list_for_each_entry_safe(uobj, tmp, &context->xrcd_list, list) {
330 struct ib_xrcd *xrcd = uobj->object;
331 struct ib_uxrcd_object *uxrcd =
332 container_of(uobj, struct ib_uxrcd_object, uobject);
333
334 idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj);
335 ib_uverbs_dealloc_xrcd(file->device, xrcd);
336 kfree(uxrcd);
337 }
338 mutex_unlock(&file->device->xrcd_tree_mutex);
339
340 list_for_each_entry_safe(uobj, tmp, &context->pd_list, list) {
341 struct ib_pd *pd = uobj->object;
342
343 idr_remove_uobj(&ib_uverbs_pd_idr, uobj);
344 ib_dealloc_pd(pd);
345 kfree(uobj);
346 }
347
348 put_pid(context->tgid);
349
350 return context->device->dealloc_ucontext(context);
351 }
352
ib_uverbs_comp_dev(struct ib_uverbs_device * dev)353 static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev)
354 {
355 complete(&dev->comp);
356 }
357
ib_uverbs_release_file(struct kref * ref)358 static void ib_uverbs_release_file(struct kref *ref)
359 {
360 struct ib_uverbs_file *file =
361 container_of(ref, struct ib_uverbs_file, ref);
362 struct ib_device *ib_dev;
363 int srcu_key;
364
365 srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
366 ib_dev = srcu_dereference(file->device->ib_dev,
367 &file->device->disassociate_srcu);
368 if (ib_dev && !ib_dev->disassociate_ucontext)
369 module_put(ib_dev->owner);
370 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
371
372 if (atomic_dec_and_test(&file->device->refcount))
373 ib_uverbs_comp_dev(file->device);
374
375 kfree(file);
376 }
377
ib_uverbs_event_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)378 static ssize_t ib_uverbs_event_read(struct file *filp, char __user *buf,
379 size_t count, loff_t *pos)
380 {
381 struct ib_uverbs_event_file *file = filp->private_data;
382 struct ib_uverbs_event *event;
383 int eventsz;
384 int ret = 0;
385
386 spin_lock_irq(&file->lock);
387
388 while (list_empty(&file->event_list)) {
389 spin_unlock_irq(&file->lock);
390
391 if (filp->f_flags & O_NONBLOCK)
392 return -EAGAIN;
393
394 if (wait_event_interruptible(file->poll_wait,
395 (!list_empty(&file->event_list) ||
396 /* The barriers built into wait_event_interruptible()
397 * and wake_up() guarentee this will see the null set
398 * without using RCU
399 */
400 !file->uverbs_file->device->ib_dev)))
401 return -ERESTARTSYS;
402
403 /* If device was disassociated and no event exists set an error */
404 if (list_empty(&file->event_list) &&
405 !file->uverbs_file->device->ib_dev)
406 return -EIO;
407
408 spin_lock_irq(&file->lock);
409 }
410
411 event = list_entry(file->event_list.next, struct ib_uverbs_event, list);
412
413 if (file->is_async)
414 eventsz = sizeof (struct ib_uverbs_async_event_desc);
415 else
416 eventsz = sizeof (struct ib_uverbs_comp_event_desc);
417
418 if (eventsz > count) {
419 ret = -EINVAL;
420 event = NULL;
421 } else {
422 list_del(file->event_list.next);
423 if (event->counter) {
424 ++(*event->counter);
425 list_del(&event->obj_list);
426 }
427 }
428
429 spin_unlock_irq(&file->lock);
430
431 if (event) {
432 if (copy_to_user(buf, event, eventsz))
433 ret = -EFAULT;
434 else
435 ret = eventsz;
436 }
437
438 kfree(event);
439
440 return ret;
441 }
442
ib_uverbs_event_poll(struct file * filp,struct poll_table_struct * wait)443 static unsigned int ib_uverbs_event_poll(struct file *filp,
444 struct poll_table_struct *wait)
445 {
446 unsigned int pollflags = 0;
447 struct ib_uverbs_event_file *file = filp->private_data;
448
449 poll_wait(filp, &file->poll_wait, wait);
450
451 spin_lock_irq(&file->lock);
452 if (!list_empty(&file->event_list))
453 pollflags = POLLIN | POLLRDNORM;
454 spin_unlock_irq(&file->lock);
455
456 return pollflags;
457 }
458
ib_uverbs_event_fasync(int fd,struct file * filp,int on)459 static int ib_uverbs_event_fasync(int fd, struct file *filp, int on)
460 {
461 struct ib_uverbs_event_file *file = filp->private_data;
462
463 return fasync_helper(fd, filp, on, &file->async_queue);
464 }
465
ib_uverbs_event_close(struct inode * inode,struct file * filp)466 static int ib_uverbs_event_close(struct inode *inode, struct file *filp)
467 {
468 struct ib_uverbs_event_file *file = filp->private_data;
469 struct ib_uverbs_event *entry, *tmp;
470 int closed_already = 0;
471
472 mutex_lock(&file->uverbs_file->device->lists_mutex);
473 spin_lock_irq(&file->lock);
474 closed_already = file->is_closed;
475 file->is_closed = 1;
476 list_for_each_entry_safe(entry, tmp, &file->event_list, list) {
477 if (entry->counter)
478 list_del(&entry->obj_list);
479 kfree(entry);
480 }
481 spin_unlock_irq(&file->lock);
482 if (!closed_already) {
483 list_del(&file->list);
484 if (file->is_async)
485 ib_unregister_event_handler(&file->uverbs_file->
486 event_handler);
487 }
488 mutex_unlock(&file->uverbs_file->device->lists_mutex);
489
490 kref_put(&file->uverbs_file->ref, ib_uverbs_release_file);
491 kref_put(&file->ref, ib_uverbs_release_event_file);
492
493 return 0;
494 }
495
496 static const struct file_operations uverbs_event_fops = {
497 .owner = THIS_MODULE,
498 .read = ib_uverbs_event_read,
499 .poll = ib_uverbs_event_poll,
500 .release = ib_uverbs_event_close,
501 .fasync = ib_uverbs_event_fasync,
502 .llseek = no_llseek,
503 };
504
ib_uverbs_comp_handler(struct ib_cq * cq,void * cq_context)505 void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context)
506 {
507 struct ib_uverbs_event_file *file = cq_context;
508 struct ib_ucq_object *uobj;
509 struct ib_uverbs_event *entry;
510 unsigned long flags;
511
512 if (!file)
513 return;
514
515 spin_lock_irqsave(&file->lock, flags);
516 if (file->is_closed) {
517 spin_unlock_irqrestore(&file->lock, flags);
518 return;
519 }
520
521 entry = kmalloc(sizeof *entry, GFP_ATOMIC);
522 if (!entry) {
523 spin_unlock_irqrestore(&file->lock, flags);
524 return;
525 }
526
527 uobj = container_of(cq->uobject, struct ib_ucq_object, uobject);
528
529 entry->desc.comp.cq_handle = cq->uobject->user_handle;
530 entry->counter = &uobj->comp_events_reported;
531
532 list_add_tail(&entry->list, &file->event_list);
533 list_add_tail(&entry->obj_list, &uobj->comp_list);
534 spin_unlock_irqrestore(&file->lock, flags);
535
536 wake_up_interruptible(&file->poll_wait);
537 kill_fasync(&file->async_queue, SIGIO, POLL_IN);
538 }
539
ib_uverbs_async_handler(struct ib_uverbs_file * file,__u64 element,__u64 event,struct list_head * obj_list,u32 * counter)540 static void ib_uverbs_async_handler(struct ib_uverbs_file *file,
541 __u64 element, __u64 event,
542 struct list_head *obj_list,
543 u32 *counter)
544 {
545 struct ib_uverbs_event *entry;
546 unsigned long flags;
547
548 spin_lock_irqsave(&file->async_file->lock, flags);
549 if (file->async_file->is_closed) {
550 spin_unlock_irqrestore(&file->async_file->lock, flags);
551 return;
552 }
553
554 entry = kmalloc(sizeof *entry, GFP_ATOMIC);
555 if (!entry) {
556 spin_unlock_irqrestore(&file->async_file->lock, flags);
557 return;
558 }
559
560 entry->desc.async.element = element;
561 entry->desc.async.event_type = event;
562 entry->desc.async.reserved = 0;
563 entry->counter = counter;
564
565 list_add_tail(&entry->list, &file->async_file->event_list);
566 if (obj_list)
567 list_add_tail(&entry->obj_list, obj_list);
568 spin_unlock_irqrestore(&file->async_file->lock, flags);
569
570 wake_up_interruptible(&file->async_file->poll_wait);
571 kill_fasync(&file->async_file->async_queue, SIGIO, POLL_IN);
572 }
573
ib_uverbs_cq_event_handler(struct ib_event * event,void * context_ptr)574 void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr)
575 {
576 struct ib_ucq_object *uobj = container_of(event->element.cq->uobject,
577 struct ib_ucq_object, uobject);
578
579 ib_uverbs_async_handler(uobj->uverbs_file, uobj->uobject.user_handle,
580 event->event, &uobj->async_list,
581 &uobj->async_events_reported);
582 }
583
ib_uverbs_qp_event_handler(struct ib_event * event,void * context_ptr)584 void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr)
585 {
586 struct ib_uevent_object *uobj;
587
588 /* for XRC target qp's, check that qp is live */
589 if (!event->element.qp->uobject || !event->element.qp->uobject->live)
590 return;
591
592 uobj = container_of(event->element.qp->uobject,
593 struct ib_uevent_object, uobject);
594
595 ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle,
596 event->event, &uobj->event_list,
597 &uobj->events_reported);
598 }
599
ib_uverbs_wq_event_handler(struct ib_event * event,void * context_ptr)600 void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr)
601 {
602 struct ib_uevent_object *uobj = container_of(event->element.wq->uobject,
603 struct ib_uevent_object, uobject);
604
605 ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle,
606 event->event, &uobj->event_list,
607 &uobj->events_reported);
608 }
609
ib_uverbs_srq_event_handler(struct ib_event * event,void * context_ptr)610 void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr)
611 {
612 struct ib_uevent_object *uobj;
613
614 uobj = container_of(event->element.srq->uobject,
615 struct ib_uevent_object, uobject);
616
617 ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle,
618 event->event, &uobj->event_list,
619 &uobj->events_reported);
620 }
621
ib_uverbs_event_handler(struct ib_event_handler * handler,struct ib_event * event)622 void ib_uverbs_event_handler(struct ib_event_handler *handler,
623 struct ib_event *event)
624 {
625 struct ib_uverbs_file *file =
626 container_of(handler, struct ib_uverbs_file, event_handler);
627
628 ib_uverbs_async_handler(file, event->element.port_num, event->event,
629 NULL, NULL);
630 }
631
ib_uverbs_free_async_event_file(struct ib_uverbs_file * file)632 void ib_uverbs_free_async_event_file(struct ib_uverbs_file *file)
633 {
634 kref_put(&file->async_file->ref, ib_uverbs_release_event_file);
635 file->async_file = NULL;
636 }
637
ib_uverbs_alloc_event_file(struct ib_uverbs_file * uverbs_file,struct ib_device * ib_dev,int is_async)638 struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file,
639 struct ib_device *ib_dev,
640 int is_async)
641 {
642 struct ib_uverbs_event_file *ev_file;
643 struct file *filp;
644 int ret;
645
646 ev_file = kzalloc(sizeof(*ev_file), GFP_KERNEL);
647 if (!ev_file)
648 return ERR_PTR(-ENOMEM);
649
650 kref_init(&ev_file->ref);
651 spin_lock_init(&ev_file->lock);
652 INIT_LIST_HEAD(&ev_file->event_list);
653 init_waitqueue_head(&ev_file->poll_wait);
654 ev_file->uverbs_file = uverbs_file;
655 kref_get(&ev_file->uverbs_file->ref);
656 ev_file->async_queue = NULL;
657 ev_file->is_closed = 0;
658
659 /*
660 * fops_get() can't fail here, because we're coming from a
661 * system call on a uverbs file, which will already have a
662 * module reference.
663 */
664 filp = alloc_file(FMODE_READ, fops_get(&uverbs_event_fops));
665 if (IS_ERR(filp))
666 goto err_put_refs;
667 filp->private_data = ev_file;
668
669 mutex_lock(&uverbs_file->device->lists_mutex);
670 list_add_tail(&ev_file->list,
671 &uverbs_file->device->uverbs_events_file_list);
672 mutex_unlock(&uverbs_file->device->lists_mutex);
673
674 if (is_async) {
675 WARN_ON(uverbs_file->async_file);
676 uverbs_file->async_file = ev_file;
677 kref_get(&uverbs_file->async_file->ref);
678 INIT_IB_EVENT_HANDLER(&uverbs_file->event_handler,
679 ib_dev,
680 ib_uverbs_event_handler);
681 ret = ib_register_event_handler(&uverbs_file->event_handler);
682 if (ret)
683 goto err_put_file;
684
685 /* At that point async file stuff was fully set */
686 ev_file->is_async = 1;
687 }
688
689 return filp;
690
691 err_put_file:
692 fput(filp);
693 kref_put(&uverbs_file->async_file->ref, ib_uverbs_release_event_file);
694 uverbs_file->async_file = NULL;
695 return ERR_PTR(ret);
696
697 err_put_refs:
698 kref_put(&ev_file->uverbs_file->ref, ib_uverbs_release_file);
699 kref_put(&ev_file->ref, ib_uverbs_release_event_file);
700 return filp;
701 }
702
703 /*
704 * Look up a completion event file by FD. If lookup is successful,
705 * takes a ref to the event file struct that it returns; if
706 * unsuccessful, returns NULL.
707 */
ib_uverbs_lookup_comp_file(int fd)708 struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd)
709 {
710 struct ib_uverbs_event_file *ev_file = NULL;
711 struct fd f = fdget(fd);
712
713 if (!f.file)
714 return NULL;
715
716 if (f.file->f_op != &uverbs_event_fops)
717 goto out;
718
719 ev_file = f.file->private_data;
720 if (ev_file->is_async) {
721 ev_file = NULL;
722 goto out;
723 }
724
725 kref_get(&ev_file->ref);
726
727 out:
728 fdput(f);
729 return ev_file;
730 }
731
verify_command_mask(struct ib_device * ib_dev,__u32 command)732 static int verify_command_mask(struct ib_device *ib_dev, __u32 command)
733 {
734 u64 mask;
735
736 if (command <= IB_USER_VERBS_CMD_OPEN_QP)
737 mask = ib_dev->uverbs_cmd_mask;
738 else
739 mask = ib_dev->uverbs_ex_cmd_mask;
740
741 if (mask & ((u64)1 << command))
742 return 0;
743
744 return -1;
745 }
746
ib_uverbs_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)747 static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
748 size_t count, loff_t *pos)
749 {
750 struct ib_uverbs_file *file = filp->private_data;
751 struct ib_device *ib_dev;
752 struct ib_uverbs_cmd_hdr hdr;
753 __u32 command;
754 __u32 flags;
755 int srcu_key;
756 ssize_t ret;
757
758 if (WARN_ON_ONCE(!ib_safe_file_access(filp)))
759 return -EACCES;
760
761 if (count < sizeof hdr)
762 return -EINVAL;
763
764 if (copy_from_user(&hdr, buf, sizeof hdr))
765 return -EFAULT;
766
767 srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
768 ib_dev = srcu_dereference(file->device->ib_dev,
769 &file->device->disassociate_srcu);
770 if (!ib_dev) {
771 ret = -EIO;
772 goto out;
773 }
774
775 if (hdr.command & ~(__u32)(IB_USER_VERBS_CMD_FLAGS_MASK |
776 IB_USER_VERBS_CMD_COMMAND_MASK)) {
777 ret = -EINVAL;
778 goto out;
779 }
780
781 command = hdr.command & IB_USER_VERBS_CMD_COMMAND_MASK;
782 if (verify_command_mask(ib_dev, command)) {
783 ret = -EOPNOTSUPP;
784 goto out;
785 }
786
787 if (!file->ucontext &&
788 command != IB_USER_VERBS_CMD_GET_CONTEXT) {
789 ret = -EINVAL;
790 goto out;
791 }
792
793 flags = (hdr.command &
794 IB_USER_VERBS_CMD_FLAGS_MASK) >> IB_USER_VERBS_CMD_FLAGS_SHIFT;
795
796 if (!flags) {
797 if (command >= ARRAY_SIZE(uverbs_cmd_table) ||
798 !uverbs_cmd_table[command]) {
799 ret = -EINVAL;
800 goto out;
801 }
802
803 if (hdr.in_words * 4 != count) {
804 ret = -EINVAL;
805 goto out;
806 }
807
808 ret = uverbs_cmd_table[command](file, ib_dev,
809 buf + sizeof(hdr),
810 hdr.in_words * 4,
811 hdr.out_words * 4);
812
813 } else if (flags == IB_USER_VERBS_CMD_FLAG_EXTENDED) {
814 struct ib_uverbs_ex_cmd_hdr ex_hdr;
815 struct ib_udata ucore;
816 struct ib_udata uhw;
817 size_t written_count = count;
818
819 if (command >= ARRAY_SIZE(uverbs_ex_cmd_table) ||
820 !uverbs_ex_cmd_table[command]) {
821 ret = -ENOSYS;
822 goto out;
823 }
824
825 if (!file->ucontext) {
826 ret = -EINVAL;
827 goto out;
828 }
829
830 if (count < (sizeof(hdr) + sizeof(ex_hdr))) {
831 ret = -EINVAL;
832 goto out;
833 }
834
835 if (copy_from_user(&ex_hdr, buf + sizeof(hdr), sizeof(ex_hdr))) {
836 ret = -EFAULT;
837 goto out;
838 }
839
840 count -= sizeof(hdr) + sizeof(ex_hdr);
841 buf += sizeof(hdr) + sizeof(ex_hdr);
842
843 if ((hdr.in_words + ex_hdr.provider_in_words) * 8 != count) {
844 ret = -EINVAL;
845 goto out;
846 }
847
848 if (ex_hdr.cmd_hdr_reserved) {
849 ret = -EINVAL;
850 goto out;
851 }
852
853 if (ex_hdr.response) {
854 if (!hdr.out_words && !ex_hdr.provider_out_words) {
855 ret = -EINVAL;
856 goto out;
857 }
858
859 if (!access_ok(VERIFY_WRITE,
860 (void __user *) (unsigned long) ex_hdr.response,
861 (hdr.out_words + ex_hdr.provider_out_words) * 8)) {
862 ret = -EFAULT;
863 goto out;
864 }
865 } else {
866 if (hdr.out_words || ex_hdr.provider_out_words) {
867 ret = -EINVAL;
868 goto out;
869 }
870 }
871
872 INIT_UDATA_BUF_OR_NULL(&ucore, buf, (unsigned long) ex_hdr.response,
873 hdr.in_words * 8, hdr.out_words * 8);
874
875 INIT_UDATA_BUF_OR_NULL(&uhw,
876 buf + ucore.inlen,
877 (unsigned long) ex_hdr.response + ucore.outlen,
878 ex_hdr.provider_in_words * 8,
879 ex_hdr.provider_out_words * 8);
880
881 ret = uverbs_ex_cmd_table[command](file,
882 ib_dev,
883 &ucore,
884 &uhw);
885 if (!ret)
886 ret = written_count;
887 } else {
888 ret = -ENOSYS;
889 }
890
891 out:
892 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
893 return ret;
894 }
895
ib_uverbs_mmap(struct file * filp,struct vm_area_struct * vma)896 static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)
897 {
898 struct ib_uverbs_file *file = filp->private_data;
899 struct ib_device *ib_dev;
900 int ret = 0;
901 int srcu_key;
902
903 srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
904 ib_dev = srcu_dereference(file->device->ib_dev,
905 &file->device->disassociate_srcu);
906 if (!ib_dev) {
907 ret = -EIO;
908 goto out;
909 }
910
911 if (!file->ucontext)
912 ret = -ENODEV;
913 else
914 ret = ib_dev->mmap(file->ucontext, vma);
915 out:
916 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
917 return ret;
918 }
919
920 /*
921 * ib_uverbs_open() does not need the BKL:
922 *
923 * - the ib_uverbs_device structures are properly reference counted and
924 * everything else is purely local to the file being created, so
925 * races against other open calls are not a problem;
926 * - there is no ioctl method to race against;
927 * - the open method will either immediately run -ENXIO, or all
928 * required initialization will be done.
929 */
ib_uverbs_open(struct inode * inode,struct file * filp)930 static int ib_uverbs_open(struct inode *inode, struct file *filp)
931 {
932 struct ib_uverbs_device *dev;
933 struct ib_uverbs_file *file;
934 struct ib_device *ib_dev;
935 int ret;
936 int module_dependent;
937 int srcu_key;
938
939 dev = container_of(inode->i_cdev->si_drv1, struct ib_uverbs_device, cdev);
940 if (!atomic_inc_not_zero(&dev->refcount))
941 return -ENXIO;
942
943 srcu_key = srcu_read_lock(&dev->disassociate_srcu);
944 mutex_lock(&dev->lists_mutex);
945 ib_dev = srcu_dereference(dev->ib_dev,
946 &dev->disassociate_srcu);
947 if (!ib_dev) {
948 ret = -EIO;
949 goto err;
950 }
951
952 /* In case IB device supports disassociate ucontext, there is no hard
953 * dependency between uverbs device and its low level device.
954 */
955 module_dependent = !(ib_dev->disassociate_ucontext);
956
957 if (module_dependent) {
958 if (!try_module_get(ib_dev->owner)) {
959 ret = -ENODEV;
960 goto err;
961 }
962 }
963
964 file = kzalloc(sizeof(*file), GFP_KERNEL);
965 if (!file) {
966 ret = -ENOMEM;
967 if (module_dependent)
968 goto err_module;
969
970 goto err;
971 }
972
973 file->device = dev;
974 file->ucontext = NULL;
975 file->async_file = NULL;
976 kref_init(&file->ref);
977 mutex_init(&file->mutex);
978 mutex_init(&file->cleanup_mutex);
979
980 filp->private_data = file;
981 kobject_get(&dev->kobj);
982 list_add_tail(&file->list, &dev->uverbs_file_list);
983 mutex_unlock(&dev->lists_mutex);
984 srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
985
986 return nonseekable_open(inode, filp);
987
988 err_module:
989 module_put(ib_dev->owner);
990
991 err:
992 mutex_unlock(&dev->lists_mutex);
993 srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
994 if (atomic_dec_and_test(&dev->refcount))
995 ib_uverbs_comp_dev(dev);
996
997 return ret;
998 }
999
ib_uverbs_close(struct inode * inode,struct file * filp)1000 static int ib_uverbs_close(struct inode *inode, struct file *filp)
1001 {
1002 struct ib_uverbs_file *file = filp->private_data;
1003 struct ib_uverbs_device *dev = file->device;
1004
1005 mutex_lock(&file->cleanup_mutex);
1006 if (file->ucontext) {
1007 ib_uverbs_cleanup_ucontext(file, file->ucontext);
1008 file->ucontext = NULL;
1009 }
1010 mutex_unlock(&file->cleanup_mutex);
1011
1012 mutex_lock(&file->device->lists_mutex);
1013 if (!file->is_closed) {
1014 list_del(&file->list);
1015 file->is_closed = 1;
1016 }
1017 mutex_unlock(&file->device->lists_mutex);
1018
1019 if (file->async_file)
1020 kref_put(&file->async_file->ref, ib_uverbs_release_event_file);
1021
1022 kref_put(&file->ref, ib_uverbs_release_file);
1023 kobject_put(&dev->kobj);
1024
1025 return 0;
1026 }
1027
1028 static const struct file_operations uverbs_fops = {
1029 .owner = THIS_MODULE,
1030 .write = ib_uverbs_write,
1031 .open = ib_uverbs_open,
1032 .release = ib_uverbs_close,
1033 .llseek = no_llseek,
1034 };
1035
1036 static const struct file_operations uverbs_mmap_fops = {
1037 .owner = THIS_MODULE,
1038 .write = ib_uverbs_write,
1039 .mmap = ib_uverbs_mmap,
1040 .open = ib_uverbs_open,
1041 .release = ib_uverbs_close,
1042 .llseek = no_llseek,
1043 };
1044
1045 static struct ib_client uverbs_client = {
1046 .name = "uverbs",
1047 .add = ib_uverbs_add_one,
1048 .remove = ib_uverbs_remove_one
1049 };
1050
show_ibdev(struct device * device,struct device_attribute * attr,char * buf)1051 static ssize_t show_ibdev(struct device *device, struct device_attribute *attr,
1052 char *buf)
1053 {
1054 int ret = -ENODEV;
1055 int srcu_key;
1056 struct ib_uverbs_device *dev = dev_get_drvdata(device);
1057 struct ib_device *ib_dev;
1058
1059 if (!dev)
1060 return -ENODEV;
1061
1062 srcu_key = srcu_read_lock(&dev->disassociate_srcu);
1063 ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1064 if (ib_dev)
1065 ret = sprintf(buf, "%s\n", ib_dev->name);
1066 srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
1067
1068 return ret;
1069 }
1070 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
1071
show_dev_abi_version(struct device * device,struct device_attribute * attr,char * buf)1072 static ssize_t show_dev_abi_version(struct device *device,
1073 struct device_attribute *attr, char *buf)
1074 {
1075 struct ib_uverbs_device *dev = dev_get_drvdata(device);
1076 int ret = -ENODEV;
1077 int srcu_key;
1078 struct ib_device *ib_dev;
1079
1080 if (!dev)
1081 return -ENODEV;
1082 srcu_key = srcu_read_lock(&dev->disassociate_srcu);
1083 ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1084 if (ib_dev)
1085 ret = sprintf(buf, "%d\n", ib_dev->uverbs_abi_ver);
1086 srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
1087
1088 return ret;
1089 }
1090 static DEVICE_ATTR(abi_version, S_IRUGO, show_dev_abi_version, NULL);
1091
1092 static CLASS_ATTR_STRING(abi_version, S_IRUGO,
1093 __stringify(IB_USER_VERBS_ABI_VERSION));
1094
1095 static dev_t overflow_maj;
1096 static DECLARE_BITMAP(overflow_map, IB_UVERBS_MAX_DEVICES);
1097
1098 /*
1099 * If we have more than IB_UVERBS_MAX_DEVICES, dynamically overflow by
1100 * requesting a new major number and doubling the number of max devices we
1101 * support. It's stupid, but simple.
1102 */
find_overflow_devnum(void)1103 static int find_overflow_devnum(void)
1104 {
1105 int ret;
1106
1107 if (!overflow_maj) {
1108 ret = alloc_chrdev_region(&overflow_maj, 0, IB_UVERBS_MAX_DEVICES,
1109 "infiniband_verbs");
1110 if (ret) {
1111 pr_err("user_verbs: couldn't register dynamic device number\n");
1112 return ret;
1113 }
1114 }
1115
1116 ret = find_first_zero_bit(overflow_map, IB_UVERBS_MAX_DEVICES);
1117 if (ret >= IB_UVERBS_MAX_DEVICES)
1118 return -1;
1119
1120 return ret;
1121 }
1122
1123 static ssize_t
show_dev_device(struct device * device,struct device_attribute * attr,char * buf)1124 show_dev_device(struct device *device, struct device_attribute *attr, char *buf)
1125 {
1126 struct ib_uverbs_device *dev = dev_get_drvdata(device);
1127
1128 if (!dev || !dev->ib_dev->dma_device)
1129 return -ENODEV;
1130
1131 return sprintf(buf, "0x%04x\n",
1132 ((struct pci_dev *)dev->ib_dev->dma_device)->device);
1133 }
1134 static DEVICE_ATTR(device, S_IRUGO, show_dev_device, NULL);
1135
1136 static ssize_t
show_dev_vendor(struct device * device,struct device_attribute * attr,char * buf)1137 show_dev_vendor(struct device *device, struct device_attribute *attr, char *buf)
1138 {
1139 struct ib_uverbs_device *dev = dev_get_drvdata(device);
1140
1141 if (!dev || !dev->ib_dev->dma_device)
1142 return -ENODEV;
1143
1144 return sprintf(buf, "0x%04x\n",
1145 ((struct pci_dev *)dev->ib_dev->dma_device)->vendor);
1146 }
1147 static DEVICE_ATTR(vendor, S_IRUGO, show_dev_vendor, NULL);
1148
1149 struct attribute *device_attrs[] =
1150 {
1151 &dev_attr_device.attr,
1152 &dev_attr_vendor.attr,
1153 NULL
1154 };
1155
1156 static struct attribute_group device_group = {
1157 .name = "device",
1158 .attrs = device_attrs
1159 };
1160
ib_uverbs_add_one(struct ib_device * device)1161 static void ib_uverbs_add_one(struct ib_device *device)
1162 {
1163 int devnum;
1164 dev_t base;
1165 struct ib_uverbs_device *uverbs_dev;
1166 int ret;
1167
1168 if (!device->alloc_ucontext)
1169 return;
1170
1171 uverbs_dev = kzalloc(sizeof *uverbs_dev, GFP_KERNEL);
1172 if (!uverbs_dev)
1173 return;
1174
1175 ret = init_srcu_struct(&uverbs_dev->disassociate_srcu);
1176 if (ret) {
1177 kfree(uverbs_dev);
1178 return;
1179 }
1180
1181 atomic_set(&uverbs_dev->refcount, 1);
1182 init_completion(&uverbs_dev->comp);
1183 uverbs_dev->xrcd_tree = RB_ROOT;
1184 mutex_init(&uverbs_dev->xrcd_tree_mutex);
1185 kobject_init(&uverbs_dev->kobj, &ib_uverbs_dev_ktype);
1186 mutex_init(&uverbs_dev->lists_mutex);
1187 INIT_LIST_HEAD(&uverbs_dev->uverbs_file_list);
1188 INIT_LIST_HEAD(&uverbs_dev->uverbs_events_file_list);
1189
1190 spin_lock(&map_lock);
1191 devnum = find_first_zero_bit(dev_map, IB_UVERBS_MAX_DEVICES);
1192 if (devnum >= IB_UVERBS_MAX_DEVICES) {
1193 spin_unlock(&map_lock);
1194 devnum = find_overflow_devnum();
1195 if (devnum < 0)
1196 goto err;
1197
1198 spin_lock(&map_lock);
1199 uverbs_dev->devnum = devnum + IB_UVERBS_MAX_DEVICES;
1200 base = devnum + overflow_maj;
1201 set_bit(devnum, overflow_map);
1202 } else {
1203 uverbs_dev->devnum = devnum;
1204 base = devnum + IB_UVERBS_BASE_DEV;
1205 set_bit(devnum, dev_map);
1206 }
1207 spin_unlock(&map_lock);
1208
1209 rcu_assign_pointer(uverbs_dev->ib_dev, device);
1210 uverbs_dev->num_comp_vectors = device->num_comp_vectors;
1211
1212 cdev_init(&uverbs_dev->cdev, NULL);
1213 uverbs_dev->cdev.owner = THIS_MODULE;
1214 uverbs_dev->cdev.ops = device->mmap ? &uverbs_mmap_fops : &uverbs_fops;
1215 uverbs_dev->cdev.kobj.parent = &uverbs_dev->kobj;
1216 kobject_set_name(&uverbs_dev->cdev.kobj, "uverbs%d", uverbs_dev->devnum);
1217 if (cdev_add(&uverbs_dev->cdev, base, 1))
1218 goto err_cdev;
1219
1220 uverbs_dev->dev = device_create(uverbs_class, device->dma_device,
1221 uverbs_dev->cdev.dev, uverbs_dev,
1222 "uverbs%d", uverbs_dev->devnum);
1223 if (IS_ERR(uverbs_dev->dev))
1224 goto err_cdev;
1225
1226 if (device_create_file(uverbs_dev->dev, &dev_attr_ibdev))
1227 goto err_class;
1228 if (device_create_file(uverbs_dev->dev, &dev_attr_abi_version))
1229 goto err_class;
1230 if (sysfs_create_group(&uverbs_dev->dev->kobj, &device_group))
1231 goto err_class;
1232
1233 ib_set_client_data(device, &uverbs_client, uverbs_dev);
1234
1235 return;
1236
1237 err_class:
1238 device_destroy(uverbs_class, uverbs_dev->cdev.dev);
1239
1240 err_cdev:
1241 cdev_del(&uverbs_dev->cdev);
1242 if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
1243 clear_bit(devnum, dev_map);
1244 else
1245 clear_bit(devnum, overflow_map);
1246
1247 err:
1248 if (atomic_dec_and_test(&uverbs_dev->refcount))
1249 ib_uverbs_comp_dev(uverbs_dev);
1250 wait_for_completion(&uverbs_dev->comp);
1251 kobject_put(&uverbs_dev->kobj);
1252 return;
1253 }
1254
ib_uverbs_free_hw_resources(struct ib_uverbs_device * uverbs_dev,struct ib_device * ib_dev)1255 static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,
1256 struct ib_device *ib_dev)
1257 {
1258 struct ib_uverbs_file *file;
1259 struct ib_uverbs_event_file *event_file;
1260 struct ib_event event;
1261
1262 /* Pending running commands to terminate */
1263 synchronize_srcu(&uverbs_dev->disassociate_srcu);
1264 event.event = IB_EVENT_DEVICE_FATAL;
1265 event.element.port_num = 0;
1266 event.device = ib_dev;
1267
1268 mutex_lock(&uverbs_dev->lists_mutex);
1269 while (!list_empty(&uverbs_dev->uverbs_file_list)) {
1270 struct ib_ucontext *ucontext;
1271 file = list_first_entry(&uverbs_dev->uverbs_file_list,
1272 struct ib_uverbs_file, list);
1273 file->is_closed = 1;
1274 list_del(&file->list);
1275 kref_get(&file->ref);
1276 mutex_unlock(&uverbs_dev->lists_mutex);
1277
1278
1279 mutex_lock(&file->cleanup_mutex);
1280 ucontext = file->ucontext;
1281 file->ucontext = NULL;
1282 mutex_unlock(&file->cleanup_mutex);
1283
1284 /* At this point ib_uverbs_close cannot be running
1285 * ib_uverbs_cleanup_ucontext
1286 */
1287 if (ucontext) {
1288 /* We must release the mutex before going ahead and
1289 * calling disassociate_ucontext. disassociate_ucontext
1290 * might end up indirectly calling uverbs_close,
1291 * for example due to freeing the resources
1292 * (e.g mmput).
1293 */
1294 ib_uverbs_event_handler(&file->event_handler, &event);
1295 ib_dev->disassociate_ucontext(ucontext);
1296 ib_uverbs_cleanup_ucontext(file, ucontext);
1297 }
1298
1299 mutex_lock(&uverbs_dev->lists_mutex);
1300 kref_put(&file->ref, ib_uverbs_release_file);
1301 }
1302
1303 while (!list_empty(&uverbs_dev->uverbs_events_file_list)) {
1304 event_file = list_first_entry(&uverbs_dev->
1305 uverbs_events_file_list,
1306 struct ib_uverbs_event_file,
1307 list);
1308 spin_lock_irq(&event_file->lock);
1309 event_file->is_closed = 1;
1310 spin_unlock_irq(&event_file->lock);
1311
1312 list_del(&event_file->list);
1313 if (event_file->is_async) {
1314 ib_unregister_event_handler(&event_file->uverbs_file->
1315 event_handler);
1316 event_file->uverbs_file->event_handler.device = NULL;
1317 }
1318
1319 wake_up_interruptible(&event_file->poll_wait);
1320 kill_fasync(&event_file->async_queue, SIGIO, POLL_IN);
1321 }
1322 mutex_unlock(&uverbs_dev->lists_mutex);
1323 }
1324
ib_uverbs_remove_one(struct ib_device * device,void * client_data)1325 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)
1326 {
1327 struct ib_uverbs_device *uverbs_dev = client_data;
1328 int wait_clients = 1;
1329
1330 if (!uverbs_dev)
1331 return;
1332
1333 sysfs_remove_group(&uverbs_dev->dev->kobj, &device_group);
1334 dev_set_drvdata(uverbs_dev->dev, NULL);
1335 device_destroy(uverbs_class, uverbs_dev->cdev.dev);
1336 cdev_del(&uverbs_dev->cdev);
1337
1338 if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
1339 clear_bit(uverbs_dev->devnum, dev_map);
1340 else
1341 clear_bit(uverbs_dev->devnum - IB_UVERBS_MAX_DEVICES, overflow_map);
1342
1343 if (device->disassociate_ucontext) {
1344 /* We disassociate HW resources and immediately return.
1345 * Userspace will see a EIO errno for all future access.
1346 * Upon returning, ib_device may be freed internally and is not
1347 * valid any more.
1348 * uverbs_device is still available until all clients close
1349 * their files, then the uverbs device ref count will be zero
1350 * and its resources will be freed.
1351 * Note: At this point no more files can be opened since the
1352 * cdev was deleted, however active clients can still issue
1353 * commands and close their open files.
1354 */
1355 rcu_assign_pointer(uverbs_dev->ib_dev, NULL);
1356 ib_uverbs_free_hw_resources(uverbs_dev, device);
1357 wait_clients = 0;
1358 }
1359
1360 if (atomic_dec_and_test(&uverbs_dev->refcount))
1361 ib_uverbs_comp_dev(uverbs_dev);
1362 if (wait_clients)
1363 wait_for_completion(&uverbs_dev->comp);
1364 kobject_put(&uverbs_dev->kobj);
1365 }
1366
uverbs_devnode(struct device * dev,umode_t * mode)1367 static char *uverbs_devnode(struct device *dev, umode_t *mode)
1368 {
1369 if (mode)
1370 *mode = 0666;
1371 return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
1372 }
1373
ib_uverbs_init(void)1374 static int __init ib_uverbs_init(void)
1375 {
1376 int ret;
1377
1378 ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES,
1379 "infiniband_verbs");
1380 if (ret) {
1381 pr_err("user_verbs: couldn't register device number\n");
1382 goto out;
1383 }
1384
1385 uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");
1386 if (IS_ERR(uverbs_class)) {
1387 ret = PTR_ERR(uverbs_class);
1388 pr_err("user_verbs: couldn't create class infiniband_verbs\n");
1389 goto out_chrdev;
1390 }
1391
1392 uverbs_class->devnode = uverbs_devnode;
1393
1394 ret = class_create_file(uverbs_class, &class_attr_abi_version.attr);
1395 if (ret) {
1396 pr_err("user_verbs: couldn't create abi_version attribute\n");
1397 goto out_class;
1398 }
1399
1400 ret = ib_register_client(&uverbs_client);
1401 if (ret) {
1402 pr_err("user_verbs: couldn't register client\n");
1403 goto out_class;
1404 }
1405
1406 return 0;
1407
1408 out_class:
1409 class_destroy(uverbs_class);
1410
1411 out_chrdev:
1412 unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);
1413
1414 out:
1415 return ret;
1416 }
1417
ib_uverbs_cleanup(void)1418 static void __exit ib_uverbs_cleanup(void)
1419 {
1420 ib_unregister_client(&uverbs_client);
1421 class_destroy(uverbs_class);
1422 unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);
1423 if (overflow_maj)
1424 unregister_chrdev_region(overflow_maj, IB_UVERBS_MAX_DEVICES);
1425 idr_destroy(&ib_uverbs_pd_idr);
1426 idr_destroy(&ib_uverbs_mr_idr);
1427 idr_destroy(&ib_uverbs_mw_idr);
1428 idr_destroy(&ib_uverbs_ah_idr);
1429 idr_destroy(&ib_uverbs_cq_idr);
1430 idr_destroy(&ib_uverbs_qp_idr);
1431 idr_destroy(&ib_uverbs_srq_idr);
1432 }
1433
1434 module_init_order(ib_uverbs_init, SI_ORDER_THIRD);
1435 module_exit(ib_uverbs_cleanup);
1436