1 /*
2 * Copyright (c) 2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <machine/machine_cpu.h>
30 #include <kern/locks.h>
31 #include <kern/mpsc_queue.h>
32 #include <kern/thread.h>
33
34 #if !DEBUG && !DEVELOPMENT
35 #error "Test only file"
36 #endif
37
38 #include <sys/errno.h>
39
40 struct mpsc_test_pingpong_queue {
41 struct mpsc_daemon_queue queue;
42 struct mpsc_queue_chain link;
43 struct mpsc_test_pingpong_queue *other;
44 uint64_t *count, *end;
45 };
46
47 static void
mpsc_test_pingpong_invoke(mpsc_queue_chain_t elm,__assert_only mpsc_daemon_queue_t dq)48 mpsc_test_pingpong_invoke(mpsc_queue_chain_t elm, __assert_only mpsc_daemon_queue_t dq)
49 {
50 struct mpsc_test_pingpong_queue *q;
51 q = mpsc_queue_element(elm, struct mpsc_test_pingpong_queue, link);
52 assert(&q->queue == dq);
53
54 if (*q->count % 10000 == 0) {
55 printf("mpsc_test_pingpong: %lld asyncs left\n", *q->count);
56 }
57 if ((*q->count)-- > 0) {
58 mpsc_daemon_enqueue(&q->other->queue, &q->other->link,
59 MPSC_QUEUE_DISABLE_PREEMPTION);
60 } else {
61 *q->end = mach_absolute_time();
62 thread_wakeup(&mpsc_test_pingpong_invoke);
63 }
64 }
65
66 /*
67 * The point of this test is to exercise the enqueue/unlock-drain race
68 * since the MPSC queue tries to mimize wakeups when it knows it's useless.
69 *
70 * It also ensures basic enqueue properties,
71 * and will panic if anything goes wrong to help debugging state.
72 *
73 * Performance wise, we will always go through the wakeup codepath,
74 * hence this is mostly a benchmark of
75 * assert_wait()/clear_wait()/thread_block()/thread_wakeup()
76 * rather than a benchmark of the MPSC queues.
77 */
78 int
mpsc_test_pingpong(uint64_t count,uint64_t * out)79 mpsc_test_pingpong(uint64_t count, uint64_t *out)
80 {
81 struct mpsc_test_pingpong_queue ping, pong;
82 kern_return_t kr;
83 wait_result_t wr;
84 uint32_t timeout = 5;
85
86 if (count < 1000 || count > 1000 * 1000) {
87 return EINVAL;
88 }
89
90 printf("mpsc_test_pingpong: START\n");
91
92 kr = mpsc_daemon_queue_init_with_thread(&ping.queue,
93 mpsc_test_pingpong_invoke, MINPRI_KERNEL, "ping",
94 MPSC_DAEMON_INIT_NONE);
95 if (kr != KERN_SUCCESS) {
96 panic("mpsc_test_pingpong: unable to create pong: %x", kr);
97 }
98
99 kr = mpsc_daemon_queue_init_with_thread(&pong.queue,
100 mpsc_test_pingpong_invoke, MINPRI_KERNEL, "pong",
101 MPSC_DAEMON_INIT_NONE);
102
103 if (kr != KERN_SUCCESS) {
104 panic("mpsc_test_pingpong: unable to create ping: %x", kr);
105 }
106
107 uint64_t n = count, start, end;
108 ping.count = pong.count = &n;
109 ping.end = pong.end = &end;
110 ping.other = &pong;
111 pong.other = &ping;
112
113 #if KASAN
114 timeout = 30;
115 #endif
116
117 assert_wait_timeout(&mpsc_test_pingpong_invoke, THREAD_UNINT,
118 timeout, NSEC_PER_SEC);
119 start = mach_absolute_time();
120 mpsc_daemon_enqueue(&ping.queue, &ping.link, MPSC_QUEUE_DISABLE_PREEMPTION);
121
122 wr = thread_block(THREAD_CONTINUE_NULL);
123 if (wr == THREAD_TIMED_OUT) {
124 panic("mpsc_test_pingpong: timed out: ping:%p pong:%p", &ping, &pong);
125 }
126
127 printf("mpsc_test_pingpong: CLEANUP\n");
128
129 mpsc_daemon_queue_cancel_and_wait(&ping.queue);
130 mpsc_daemon_queue_cancel_and_wait(&pong.queue);
131 absolutetime_to_nanoseconds(end - start, out);
132
133 printf("mpsc_test_pingpong: %lld ping-pongs in %lld ns (%lld.%03lld us/async)\n",
134 count, *out, (*out / count) / 1000, (*out / count) % 1000);
135 return 0;
136 }
137