xref: /linux-6.15/drivers/misc/ntsync.c (revision 31ca7bb8)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * ntsync.c - Kernel driver for NT synchronization primitives
4  *
5  * Copyright (C) 2024 Elizabeth Figura <[email protected]>
6  */
7 
8 #include <linux/anon_inodes.h>
9 #include <linux/atomic.h>
10 #include <linux/file.h>
11 #include <linux/fs.h>
12 #include <linux/hrtimer.h>
13 #include <linux/ktime.h>
14 #include <linux/miscdevice.h>
15 #include <linux/module.h>
16 #include <linux/mutex.h>
17 #include <linux/overflow.h>
18 #include <linux/sched.h>
19 #include <linux/sched/signal.h>
20 #include <linux/slab.h>
21 #include <linux/spinlock.h>
22 #include <uapi/linux/ntsync.h>
23 
24 #define NTSYNC_NAME	"ntsync"
25 
26 enum ntsync_type {
27 	NTSYNC_TYPE_SEM,
28 	NTSYNC_TYPE_MUTEX,
29 };
30 
31 /*
32  * Individual synchronization primitives are represented by
33  * struct ntsync_obj, and each primitive is backed by a file.
34  *
35  * The whole namespace is represented by a struct ntsync_device also
36  * backed by a file.
37  *
38  * Both rely on struct file for reference counting. Individual
39  * ntsync_obj objects take a reference to the device when created.
40  * Wait operations take a reference to each object being waited on for
41  * the duration of the wait.
42  */
43 
44 struct ntsync_obj {
45 	spinlock_t lock;
46 	int dev_locked;
47 
48 	enum ntsync_type type;
49 
50 	struct file *file;
51 	struct ntsync_device *dev;
52 
53 	/* The following fields are protected by the object lock. */
54 	union {
55 		struct {
56 			__u32 count;
57 			__u32 max;
58 		} sem;
59 		struct {
60 			__u32 count;
61 			pid_t owner;
62 		} mutex;
63 	} u;
64 
65 	/*
66 	 * any_waiters is protected by the object lock, but all_waiters is
67 	 * protected by the device wait_all_lock.
68 	 */
69 	struct list_head any_waiters;
70 	struct list_head all_waiters;
71 
72 	/*
73 	 * Hint describing how many tasks are queued on this object in a
74 	 * wait-all operation.
75 	 *
76 	 * Any time we do a wake, we may need to wake "all" waiters as well as
77 	 * "any" waiters. In order to atomically wake "all" waiters, we must
78 	 * lock all of the objects, and that means grabbing the wait_all_lock
79 	 * below (and, due to lock ordering rules, before locking this object).
80 	 * However, wait-all is a rare operation, and grabbing the wait-all
81 	 * lock for every wake would create unnecessary contention.
82 	 * Therefore we first check whether all_hint is zero, and, if it is,
83 	 * we skip trying to wake "all" waiters.
84 	 *
85 	 * Since wait requests must originate from user-space threads, we're
86 	 * limited here by PID_MAX_LIMIT, so there's no risk of overflow.
87 	 */
88 	atomic_t all_hint;
89 };
90 
91 struct ntsync_q_entry {
92 	struct list_head node;
93 	struct ntsync_q *q;
94 	struct ntsync_obj *obj;
95 	__u32 index;
96 };
97 
98 struct ntsync_q {
99 	struct task_struct *task;
100 	__u32 owner;
101 
102 	/*
103 	 * Protected via atomic_try_cmpxchg(). Only the thread that wins the
104 	 * compare-and-swap may actually change object states and wake this
105 	 * task.
106 	 */
107 	atomic_t signaled;
108 
109 	bool all;
110 	__u32 count;
111 	struct ntsync_q_entry entries[];
112 };
113 
114 struct ntsync_device {
115 	/*
116 	 * Wait-all operations must atomically grab all objects, and be totally
117 	 * ordered with respect to each other and wait-any operations.
118 	 * If one thread is trying to acquire several objects, another thread
119 	 * cannot touch the object at the same time.
120 	 *
121 	 * This device-wide lock is used to serialize wait-for-all
122 	 * operations, and operations on an object that is involved in a
123 	 * wait-for-all.
124 	 */
125 	struct mutex wait_all_lock;
126 
127 	struct file *file;
128 };
129 
130 /*
131  * Single objects are locked using obj->lock.
132  *
133  * Multiple objects are 'locked' while holding dev->wait_all_lock.
134  * In this case however, individual objects are not locked by holding
135  * obj->lock, but by setting obj->dev_locked.
136  *
137  * This means that in order to lock a single object, the sequence is slightly
138  * more complicated than usual. Specifically it needs to check obj->dev_locked
139  * after acquiring obj->lock, if set, it needs to drop the lock and acquire
140  * dev->wait_all_lock in order to serialize against the multi-object operation.
141  */
142 
143 static void dev_lock_obj(struct ntsync_device *dev, struct ntsync_obj *obj)
144 {
145 	lockdep_assert_held(&dev->wait_all_lock);
146 	lockdep_assert(obj->dev == dev);
147 	spin_lock(&obj->lock);
148 	/*
149 	 * By setting obj->dev_locked inside obj->lock, it is ensured that
150 	 * anyone holding obj->lock must see the value.
151 	 */
152 	obj->dev_locked = 1;
153 	spin_unlock(&obj->lock);
154 }
155 
156 static void dev_unlock_obj(struct ntsync_device *dev, struct ntsync_obj *obj)
157 {
158 	lockdep_assert_held(&dev->wait_all_lock);
159 	lockdep_assert(obj->dev == dev);
160 	spin_lock(&obj->lock);
161 	obj->dev_locked = 0;
162 	spin_unlock(&obj->lock);
163 }
164 
165 static void obj_lock(struct ntsync_obj *obj)
166 {
167 	struct ntsync_device *dev = obj->dev;
168 
169 	for (;;) {
170 		spin_lock(&obj->lock);
171 		if (likely(!obj->dev_locked))
172 			break;
173 
174 		spin_unlock(&obj->lock);
175 		mutex_lock(&dev->wait_all_lock);
176 		spin_lock(&obj->lock);
177 		/*
178 		 * obj->dev_locked should be set and released under the same
179 		 * wait_all_lock section, since we now own this lock, it should
180 		 * be clear.
181 		 */
182 		lockdep_assert(!obj->dev_locked);
183 		spin_unlock(&obj->lock);
184 		mutex_unlock(&dev->wait_all_lock);
185 	}
186 }
187 
188 static void obj_unlock(struct ntsync_obj *obj)
189 {
190 	spin_unlock(&obj->lock);
191 }
192 
193 static bool ntsync_lock_obj(struct ntsync_device *dev, struct ntsync_obj *obj)
194 {
195 	bool all;
196 
197 	obj_lock(obj);
198 	all = atomic_read(&obj->all_hint);
199 	if (unlikely(all)) {
200 		obj_unlock(obj);
201 		mutex_lock(&dev->wait_all_lock);
202 		dev_lock_obj(dev, obj);
203 	}
204 
205 	return all;
206 }
207 
208 static void ntsync_unlock_obj(struct ntsync_device *dev, struct ntsync_obj *obj, bool all)
209 {
210 	if (all) {
211 		dev_unlock_obj(dev, obj);
212 		mutex_unlock(&dev->wait_all_lock);
213 	} else {
214 		obj_unlock(obj);
215 	}
216 }
217 
218 #define ntsync_assert_held(obj) \
219 	lockdep_assert((lockdep_is_held(&(obj)->lock) != LOCK_STATE_NOT_HELD) || \
220 		       ((lockdep_is_held(&(obj)->dev->wait_all_lock) != LOCK_STATE_NOT_HELD) && \
221 			(obj)->dev_locked))
222 
223 static bool is_signaled(struct ntsync_obj *obj, __u32 owner)
224 {
225 	ntsync_assert_held(obj);
226 
227 	switch (obj->type) {
228 	case NTSYNC_TYPE_SEM:
229 		return !!obj->u.sem.count;
230 	case NTSYNC_TYPE_MUTEX:
231 		if (obj->u.mutex.owner && obj->u.mutex.owner != owner)
232 			return false;
233 		return obj->u.mutex.count < UINT_MAX;
234 	}
235 
236 	WARN(1, "bad object type %#x\n", obj->type);
237 	return false;
238 }
239 
240 /*
241  * "locked_obj" is an optional pointer to an object which is already locked and
242  * should not be locked again. This is necessary so that changing an object's
243  * state and waking it can be a single atomic operation.
244  */
245 static void try_wake_all(struct ntsync_device *dev, struct ntsync_q *q,
246 			 struct ntsync_obj *locked_obj)
247 {
248 	__u32 count = q->count;
249 	bool can_wake = true;
250 	int signaled = -1;
251 	__u32 i;
252 
253 	lockdep_assert_held(&dev->wait_all_lock);
254 	if (locked_obj)
255 		lockdep_assert(locked_obj->dev_locked);
256 
257 	for (i = 0; i < count; i++) {
258 		if (q->entries[i].obj != locked_obj)
259 			dev_lock_obj(dev, q->entries[i].obj);
260 	}
261 
262 	for (i = 0; i < count; i++) {
263 		if (!is_signaled(q->entries[i].obj, q->owner)) {
264 			can_wake = false;
265 			break;
266 		}
267 	}
268 
269 	if (can_wake && atomic_try_cmpxchg(&q->signaled, &signaled, 0)) {
270 		for (i = 0; i < count; i++) {
271 			struct ntsync_obj *obj = q->entries[i].obj;
272 
273 			switch (obj->type) {
274 			case NTSYNC_TYPE_SEM:
275 				obj->u.sem.count--;
276 				break;
277 			case NTSYNC_TYPE_MUTEX:
278 				obj->u.mutex.count++;
279 				obj->u.mutex.owner = q->owner;
280 				break;
281 			}
282 		}
283 		wake_up_process(q->task);
284 	}
285 
286 	for (i = 0; i < count; i++) {
287 		if (q->entries[i].obj != locked_obj)
288 			dev_unlock_obj(dev, q->entries[i].obj);
289 	}
290 }
291 
292 static void try_wake_all_obj(struct ntsync_device *dev, struct ntsync_obj *obj)
293 {
294 	struct ntsync_q_entry *entry;
295 
296 	lockdep_assert_held(&dev->wait_all_lock);
297 	lockdep_assert(obj->dev_locked);
298 
299 	list_for_each_entry(entry, &obj->all_waiters, node)
300 		try_wake_all(dev, entry->q, obj);
301 }
302 
303 static void try_wake_any_sem(struct ntsync_obj *sem)
304 {
305 	struct ntsync_q_entry *entry;
306 
307 	ntsync_assert_held(sem);
308 	lockdep_assert(sem->type == NTSYNC_TYPE_SEM);
309 
310 	list_for_each_entry(entry, &sem->any_waiters, node) {
311 		struct ntsync_q *q = entry->q;
312 		int signaled = -1;
313 
314 		if (!sem->u.sem.count)
315 			break;
316 
317 		if (atomic_try_cmpxchg(&q->signaled, &signaled, entry->index)) {
318 			sem->u.sem.count--;
319 			wake_up_process(q->task);
320 		}
321 	}
322 }
323 
324 static void try_wake_any_mutex(struct ntsync_obj *mutex)
325 {
326 	struct ntsync_q_entry *entry;
327 
328 	ntsync_assert_held(mutex);
329 	lockdep_assert(mutex->type == NTSYNC_TYPE_MUTEX);
330 
331 	list_for_each_entry(entry, &mutex->any_waiters, node) {
332 		struct ntsync_q *q = entry->q;
333 		int signaled = -1;
334 
335 		if (mutex->u.mutex.count == UINT_MAX)
336 			break;
337 		if (mutex->u.mutex.owner && mutex->u.mutex.owner != q->owner)
338 			continue;
339 
340 		if (atomic_try_cmpxchg(&q->signaled, &signaled, entry->index)) {
341 			mutex->u.mutex.count++;
342 			mutex->u.mutex.owner = q->owner;
343 			wake_up_process(q->task);
344 		}
345 	}
346 }
347 
348 /*
349  * Actually change the semaphore state, returning -EOVERFLOW if it is made
350  * invalid.
351  */
352 static int release_sem_state(struct ntsync_obj *sem, __u32 count)
353 {
354 	__u32 sum;
355 
356 	ntsync_assert_held(sem);
357 
358 	if (check_add_overflow(sem->u.sem.count, count, &sum) ||
359 	    sum > sem->u.sem.max)
360 		return -EOVERFLOW;
361 
362 	sem->u.sem.count = sum;
363 	return 0;
364 }
365 
366 static int ntsync_sem_release(struct ntsync_obj *sem, void __user *argp)
367 {
368 	struct ntsync_device *dev = sem->dev;
369 	__u32 __user *user_args = argp;
370 	__u32 prev_count;
371 	__u32 args;
372 	bool all;
373 	int ret;
374 
375 	if (copy_from_user(&args, argp, sizeof(args)))
376 		return -EFAULT;
377 
378 	if (sem->type != NTSYNC_TYPE_SEM)
379 		return -EINVAL;
380 
381 	all = ntsync_lock_obj(dev, sem);
382 
383 	prev_count = sem->u.sem.count;
384 	ret = release_sem_state(sem, args);
385 	if (!ret) {
386 		if (all)
387 			try_wake_all_obj(dev, sem);
388 		try_wake_any_sem(sem);
389 	}
390 
391 	ntsync_unlock_obj(dev, sem, all);
392 
393 	if (!ret && put_user(prev_count, user_args))
394 		ret = -EFAULT;
395 
396 	return ret;
397 }
398 
399 /*
400  * Actually change the mutex state, returning -EPERM if not the owner.
401  */
402 static int unlock_mutex_state(struct ntsync_obj *mutex,
403 			      const struct ntsync_mutex_args *args)
404 {
405 	ntsync_assert_held(mutex);
406 
407 	if (mutex->u.mutex.owner != args->owner)
408 		return -EPERM;
409 
410 	if (!--mutex->u.mutex.count)
411 		mutex->u.mutex.owner = 0;
412 	return 0;
413 }
414 
415 static int ntsync_mutex_unlock(struct ntsync_obj *mutex, void __user *argp)
416 {
417 	struct ntsync_mutex_args __user *user_args = argp;
418 	struct ntsync_device *dev = mutex->dev;
419 	struct ntsync_mutex_args args;
420 	__u32 prev_count;
421 	bool all;
422 	int ret;
423 
424 	if (copy_from_user(&args, argp, sizeof(args)))
425 		return -EFAULT;
426 	if (!args.owner)
427 		return -EINVAL;
428 
429 	if (mutex->type != NTSYNC_TYPE_MUTEX)
430 		return -EINVAL;
431 
432 	all = ntsync_lock_obj(dev, mutex);
433 
434 	prev_count = mutex->u.mutex.count;
435 	ret = unlock_mutex_state(mutex, &args);
436 	if (!ret) {
437 		if (all)
438 			try_wake_all_obj(dev, mutex);
439 		try_wake_any_mutex(mutex);
440 	}
441 
442 	ntsync_unlock_obj(dev, mutex, all);
443 
444 	if (!ret && put_user(prev_count, &user_args->count))
445 		ret = -EFAULT;
446 
447 	return ret;
448 }
449 
450 static int ntsync_obj_release(struct inode *inode, struct file *file)
451 {
452 	struct ntsync_obj *obj = file->private_data;
453 
454 	fput(obj->dev->file);
455 	kfree(obj);
456 
457 	return 0;
458 }
459 
460 static long ntsync_obj_ioctl(struct file *file, unsigned int cmd,
461 			     unsigned long parm)
462 {
463 	struct ntsync_obj *obj = file->private_data;
464 	void __user *argp = (void __user *)parm;
465 
466 	switch (cmd) {
467 	case NTSYNC_IOC_SEM_RELEASE:
468 		return ntsync_sem_release(obj, argp);
469 	case NTSYNC_IOC_MUTEX_UNLOCK:
470 		return ntsync_mutex_unlock(obj, argp);
471 	default:
472 		return -ENOIOCTLCMD;
473 	}
474 }
475 
476 static const struct file_operations ntsync_obj_fops = {
477 	.owner		= THIS_MODULE,
478 	.release	= ntsync_obj_release,
479 	.unlocked_ioctl	= ntsync_obj_ioctl,
480 	.compat_ioctl	= compat_ptr_ioctl,
481 };
482 
483 static struct ntsync_obj *ntsync_alloc_obj(struct ntsync_device *dev,
484 					   enum ntsync_type type)
485 {
486 	struct ntsync_obj *obj;
487 
488 	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
489 	if (!obj)
490 		return NULL;
491 	obj->type = type;
492 	obj->dev = dev;
493 	get_file(dev->file);
494 	spin_lock_init(&obj->lock);
495 	INIT_LIST_HEAD(&obj->any_waiters);
496 	INIT_LIST_HEAD(&obj->all_waiters);
497 	atomic_set(&obj->all_hint, 0);
498 
499 	return obj;
500 }
501 
502 static int ntsync_obj_get_fd(struct ntsync_obj *obj)
503 {
504 	struct file *file;
505 	int fd;
506 
507 	fd = get_unused_fd_flags(O_CLOEXEC);
508 	if (fd < 0)
509 		return fd;
510 	file = anon_inode_getfile("ntsync", &ntsync_obj_fops, obj, O_RDWR);
511 	if (IS_ERR(file)) {
512 		put_unused_fd(fd);
513 		return PTR_ERR(file);
514 	}
515 	obj->file = file;
516 	fd_install(fd, file);
517 
518 	return fd;
519 }
520 
521 static int ntsync_create_sem(struct ntsync_device *dev, void __user *argp)
522 {
523 	struct ntsync_sem_args args;
524 	struct ntsync_obj *sem;
525 	int fd;
526 
527 	if (copy_from_user(&args, argp, sizeof(args)))
528 		return -EFAULT;
529 
530 	if (args.count > args.max)
531 		return -EINVAL;
532 
533 	sem = ntsync_alloc_obj(dev, NTSYNC_TYPE_SEM);
534 	if (!sem)
535 		return -ENOMEM;
536 	sem->u.sem.count = args.count;
537 	sem->u.sem.max = args.max;
538 	fd = ntsync_obj_get_fd(sem);
539 	if (fd < 0)
540 		kfree(sem);
541 
542 	return fd;
543 }
544 
545 static int ntsync_create_mutex(struct ntsync_device *dev, void __user *argp)
546 {
547 	struct ntsync_mutex_args args;
548 	struct ntsync_obj *mutex;
549 	int fd;
550 
551 	if (copy_from_user(&args, argp, sizeof(args)))
552 		return -EFAULT;
553 
554 	if (!args.owner != !args.count)
555 		return -EINVAL;
556 
557 	mutex = ntsync_alloc_obj(dev, NTSYNC_TYPE_MUTEX);
558 	if (!mutex)
559 		return -ENOMEM;
560 	mutex->u.mutex.count = args.count;
561 	mutex->u.mutex.owner = args.owner;
562 	fd = ntsync_obj_get_fd(mutex);
563 	if (fd < 0)
564 		kfree(mutex);
565 
566 	return fd;
567 }
568 
569 static struct ntsync_obj *get_obj(struct ntsync_device *dev, int fd)
570 {
571 	struct file *file = fget(fd);
572 	struct ntsync_obj *obj;
573 
574 	if (!file)
575 		return NULL;
576 
577 	if (file->f_op != &ntsync_obj_fops) {
578 		fput(file);
579 		return NULL;
580 	}
581 
582 	obj = file->private_data;
583 	if (obj->dev != dev) {
584 		fput(file);
585 		return NULL;
586 	}
587 
588 	return obj;
589 }
590 
591 static void put_obj(struct ntsync_obj *obj)
592 {
593 	fput(obj->file);
594 }
595 
596 static int ntsync_schedule(const struct ntsync_q *q, const struct ntsync_wait_args *args)
597 {
598 	ktime_t timeout = ns_to_ktime(args->timeout);
599 	clockid_t clock = CLOCK_MONOTONIC;
600 	ktime_t *timeout_ptr;
601 	int ret = 0;
602 
603 	timeout_ptr = (args->timeout == U64_MAX ? NULL : &timeout);
604 
605 	if (args->flags & NTSYNC_WAIT_REALTIME)
606 		clock = CLOCK_REALTIME;
607 
608 	do {
609 		if (signal_pending(current)) {
610 			ret = -ERESTARTSYS;
611 			break;
612 		}
613 
614 		set_current_state(TASK_INTERRUPTIBLE);
615 		if (atomic_read(&q->signaled) != -1) {
616 			ret = 0;
617 			break;
618 		}
619 		ret = schedule_hrtimeout_range_clock(timeout_ptr, 0, HRTIMER_MODE_ABS, clock);
620 	} while (ret < 0);
621 	__set_current_state(TASK_RUNNING);
622 
623 	return ret;
624 }
625 
626 /*
627  * Allocate and initialize the ntsync_q structure, but do not queue us yet.
628  */
629 static int setup_wait(struct ntsync_device *dev,
630 		      const struct ntsync_wait_args *args, bool all,
631 		      struct ntsync_q **ret_q)
632 {
633 	const __u32 count = args->count;
634 	int fds[NTSYNC_MAX_WAIT_COUNT];
635 	struct ntsync_q *q;
636 	__u32 i, j;
637 
638 	if (args->pad[0] || args->pad[1] || (args->flags & ~NTSYNC_WAIT_REALTIME))
639 		return -EINVAL;
640 
641 	if (args->count > NTSYNC_MAX_WAIT_COUNT)
642 		return -EINVAL;
643 
644 	if (copy_from_user(fds, u64_to_user_ptr(args->objs),
645 			   array_size(count, sizeof(*fds))))
646 		return -EFAULT;
647 
648 	q = kmalloc(struct_size(q, entries, count), GFP_KERNEL);
649 	if (!q)
650 		return -ENOMEM;
651 	q->task = current;
652 	q->owner = args->owner;
653 	atomic_set(&q->signaled, -1);
654 	q->all = all;
655 	q->count = count;
656 
657 	for (i = 0; i < count; i++) {
658 		struct ntsync_q_entry *entry = &q->entries[i];
659 		struct ntsync_obj *obj = get_obj(dev, fds[i]);
660 
661 		if (!obj)
662 			goto err;
663 
664 		if (all) {
665 			/* Check that the objects are all distinct. */
666 			for (j = 0; j < i; j++) {
667 				if (obj == q->entries[j].obj) {
668 					put_obj(obj);
669 					goto err;
670 				}
671 			}
672 		}
673 
674 		entry->obj = obj;
675 		entry->q = q;
676 		entry->index = i;
677 	}
678 
679 	*ret_q = q;
680 	return 0;
681 
682 err:
683 	for (j = 0; j < i; j++)
684 		put_obj(q->entries[j].obj);
685 	kfree(q);
686 	return -EINVAL;
687 }
688 
689 static void try_wake_any_obj(struct ntsync_obj *obj)
690 {
691 	switch (obj->type) {
692 	case NTSYNC_TYPE_SEM:
693 		try_wake_any_sem(obj);
694 		break;
695 	case NTSYNC_TYPE_MUTEX:
696 		try_wake_any_mutex(obj);
697 		break;
698 	}
699 }
700 
701 static int ntsync_wait_any(struct ntsync_device *dev, void __user *argp)
702 {
703 	struct ntsync_wait_args args;
704 	struct ntsync_q *q;
705 	int signaled;
706 	bool all;
707 	__u32 i;
708 	int ret;
709 
710 	if (copy_from_user(&args, argp, sizeof(args)))
711 		return -EFAULT;
712 
713 	ret = setup_wait(dev, &args, false, &q);
714 	if (ret < 0)
715 		return ret;
716 
717 	/* queue ourselves */
718 
719 	for (i = 0; i < args.count; i++) {
720 		struct ntsync_q_entry *entry = &q->entries[i];
721 		struct ntsync_obj *obj = entry->obj;
722 
723 		all = ntsync_lock_obj(dev, obj);
724 		list_add_tail(&entry->node, &obj->any_waiters);
725 		ntsync_unlock_obj(dev, obj, all);
726 	}
727 
728 	/* check if we are already signaled */
729 
730 	for (i = 0; i < args.count; i++) {
731 		struct ntsync_obj *obj = q->entries[i].obj;
732 
733 		if (atomic_read(&q->signaled) != -1)
734 			break;
735 
736 		all = ntsync_lock_obj(dev, obj);
737 		try_wake_any_obj(obj);
738 		ntsync_unlock_obj(dev, obj, all);
739 	}
740 
741 	/* sleep */
742 
743 	ret = ntsync_schedule(q, &args);
744 
745 	/* and finally, unqueue */
746 
747 	for (i = 0; i < args.count; i++) {
748 		struct ntsync_q_entry *entry = &q->entries[i];
749 		struct ntsync_obj *obj = entry->obj;
750 
751 		all = ntsync_lock_obj(dev, obj);
752 		list_del(&entry->node);
753 		ntsync_unlock_obj(dev, obj, all);
754 
755 		put_obj(obj);
756 	}
757 
758 	signaled = atomic_read(&q->signaled);
759 	if (signaled != -1) {
760 		struct ntsync_wait_args __user *user_args = argp;
761 
762 		/* even if we caught a signal, we need to communicate success */
763 		ret = 0;
764 
765 		if (put_user(signaled, &user_args->index))
766 			ret = -EFAULT;
767 	} else if (!ret) {
768 		ret = -ETIMEDOUT;
769 	}
770 
771 	kfree(q);
772 	return ret;
773 }
774 
775 static int ntsync_wait_all(struct ntsync_device *dev, void __user *argp)
776 {
777 	struct ntsync_wait_args args;
778 	struct ntsync_q *q;
779 	int signaled;
780 	__u32 i;
781 	int ret;
782 
783 	if (copy_from_user(&args, argp, sizeof(args)))
784 		return -EFAULT;
785 
786 	ret = setup_wait(dev, &args, true, &q);
787 	if (ret < 0)
788 		return ret;
789 
790 	/* queue ourselves */
791 
792 	mutex_lock(&dev->wait_all_lock);
793 
794 	for (i = 0; i < args.count; i++) {
795 		struct ntsync_q_entry *entry = &q->entries[i];
796 		struct ntsync_obj *obj = entry->obj;
797 
798 		atomic_inc(&obj->all_hint);
799 
800 		/*
801 		 * obj->all_waiters is protected by dev->wait_all_lock rather
802 		 * than obj->lock, so there is no need to acquire obj->lock
803 		 * here.
804 		 */
805 		list_add_tail(&entry->node, &obj->all_waiters);
806 	}
807 
808 	/* check if we are already signaled */
809 
810 	try_wake_all(dev, q, NULL);
811 
812 	mutex_unlock(&dev->wait_all_lock);
813 
814 	/* sleep */
815 
816 	ret = ntsync_schedule(q, &args);
817 
818 	/* and finally, unqueue */
819 
820 	mutex_lock(&dev->wait_all_lock);
821 
822 	for (i = 0; i < args.count; i++) {
823 		struct ntsync_q_entry *entry = &q->entries[i];
824 		struct ntsync_obj *obj = entry->obj;
825 
826 		/*
827 		 * obj->all_waiters is protected by dev->wait_all_lock rather
828 		 * than obj->lock, so there is no need to acquire it here.
829 		 */
830 		list_del(&entry->node);
831 
832 		atomic_dec(&obj->all_hint);
833 
834 		put_obj(obj);
835 	}
836 
837 	mutex_unlock(&dev->wait_all_lock);
838 
839 	signaled = atomic_read(&q->signaled);
840 	if (signaled != -1) {
841 		struct ntsync_wait_args __user *user_args = argp;
842 
843 		/* even if we caught a signal, we need to communicate success */
844 		ret = 0;
845 
846 		if (put_user(signaled, &user_args->index))
847 			ret = -EFAULT;
848 	} else if (!ret) {
849 		ret = -ETIMEDOUT;
850 	}
851 
852 	kfree(q);
853 	return ret;
854 }
855 
856 static int ntsync_char_open(struct inode *inode, struct file *file)
857 {
858 	struct ntsync_device *dev;
859 
860 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
861 	if (!dev)
862 		return -ENOMEM;
863 
864 	mutex_init(&dev->wait_all_lock);
865 
866 	file->private_data = dev;
867 	dev->file = file;
868 	return nonseekable_open(inode, file);
869 }
870 
871 static int ntsync_char_release(struct inode *inode, struct file *file)
872 {
873 	struct ntsync_device *dev = file->private_data;
874 
875 	kfree(dev);
876 
877 	return 0;
878 }
879 
880 static long ntsync_char_ioctl(struct file *file, unsigned int cmd,
881 			      unsigned long parm)
882 {
883 	struct ntsync_device *dev = file->private_data;
884 	void __user *argp = (void __user *)parm;
885 
886 	switch (cmd) {
887 	case NTSYNC_IOC_CREATE_MUTEX:
888 		return ntsync_create_mutex(dev, argp);
889 	case NTSYNC_IOC_CREATE_SEM:
890 		return ntsync_create_sem(dev, argp);
891 	case NTSYNC_IOC_WAIT_ALL:
892 		return ntsync_wait_all(dev, argp);
893 	case NTSYNC_IOC_WAIT_ANY:
894 		return ntsync_wait_any(dev, argp);
895 	default:
896 		return -ENOIOCTLCMD;
897 	}
898 }
899 
900 static const struct file_operations ntsync_fops = {
901 	.owner		= THIS_MODULE,
902 	.open		= ntsync_char_open,
903 	.release	= ntsync_char_release,
904 	.unlocked_ioctl	= ntsync_char_ioctl,
905 	.compat_ioctl	= compat_ptr_ioctl,
906 };
907 
908 static struct miscdevice ntsync_misc = {
909 	.minor		= MISC_DYNAMIC_MINOR,
910 	.name		= NTSYNC_NAME,
911 	.fops		= &ntsync_fops,
912 };
913 
914 module_misc_device(ntsync_misc);
915 
916 MODULE_AUTHOR("Elizabeth Figura <[email protected]>");
917 MODULE_DESCRIPTION("Kernel driver for NT synchronization primitives");
918 MODULE_LICENSE("GPL");
919