1 #include <unistd.h>
2 #include <stdio.h>
3 #include <stdlib.h>
4 #include <fcntl.h>
5 #include <pthread.h>
6 #include <errno.h>
7 #include <err.h>
8 #include <string.h>
9 #include <assert.h>
10 #include <sysexits.h>
11 #include <getopt.h>
12 #include <spawn.h>
13 #include <stdbool.h>
14 #include <sys/sysctl.h>
15 #include <mach/mach_time.h>
16 #include <mach/mach.h>
17 #include <mach/semaphore.h>
18 #include <TargetConditionals.h>
19
20 #ifdef T_NAMESPACE
21 #undef T_NAMESPACE
22 #endif
23
24 #include <darwintest.h>
25 #include <stdatomic.h>
26
27 #define MAX_THREADS 32
28 #define SPIN_SECS 6
29 #define THR_SPINNER_PRI 63
30 #define THR_MANAGER_PRI 62
31 #define WARMUP_ITERATIONS 100
32 #define FILE_SIZE (16384 * 4096)
33 #define IO_SIZE 4096
34 #define IO_COUNT 2500
35
36 static mach_timebase_info_data_t timebase_info;
37 static semaphore_t semaphore;
38 static semaphore_t worker_sem;
39 static uint32_t g_numcpus;
40 static _Atomic uint32_t keep_going = 1;
41 int test_file_fd = 0;
42 char *data_buf = NULL;
43 extern char **environ;
44
45 static struct {
46 pthread_t thread;
47 } threads[MAX_THREADS];
48
49 static uint64_t
nanos_to_abs(uint64_t nanos)50 nanos_to_abs(uint64_t nanos)
51 {
52 return nanos * timebase_info.denom / timebase_info.numer;
53 }
54
55 static void
io_perf_test_io_init(void)56 io_perf_test_io_init(void)
57 {
58 int spawn_ret, pid;
59 char *const mount_args[] = {"/usr/local/sbin/mount_nand.sh", NULL};
60 spawn_ret = posix_spawn(&pid, mount_args[0], NULL, NULL, mount_args, environ);
61 if (spawn_ret < 0) {
62 T_SKIP("NAND mounting in LTE not possible on this device. Skipping test!");
63 }
64 waitpid(pid, &spawn_ret, 0);
65 if (WIFEXITED(spawn_ret) && !WEXITSTATUS(spawn_ret)) {
66 T_PASS("NAND mounted successfully");
67 } else {
68 T_SKIP("Unable to mount NAND. Skipping test!");
69 }
70
71 /* Mark the main thread as fixed priority */
72 struct sched_param param = {.sched_priority = THR_MANAGER_PRI};
73 T_ASSERT_POSIX_ZERO(pthread_setschedparam(pthread_self(), SCHED_FIFO, ¶m),
74 "pthread_setschedparam");
75
76 /* Set I/O Policy to Tier 0 */
77 T_ASSERT_POSIX_ZERO(setiopolicy_np(IOPOL_TYPE_DISK, IOPOL_SCOPE_PROCESS,
78 IOPOL_IMPORTANT), "setiopolicy");
79
80 /* Create data buffer */
81 data_buf = malloc(IO_SIZE * 16);
82 T_ASSERT_NOTNULL(data_buf, "Data buffer allocation");
83
84 int rndfd = open("/dev/urandom", O_RDONLY, S_IRUSR);
85 T_ASSERT_POSIX_SUCCESS(rndfd, "Open /dev/urandom");
86 T_ASSERT_GE_INT((int)read(rndfd, data_buf, IO_SIZE * 16), 0, "read /dev/urandom");
87 close(rndfd);
88
89 /* Create test file */
90 int fd = open("/mnt2/test", O_CREAT | O_WRONLY, S_IRUSR);
91 T_ASSERT_POSIX_SUCCESS(fd, 0, "Open /mnt2/test for writing!");
92
93 T_ASSERT_POSIX_ZERO(fcntl(fd, F_NOCACHE, 1), "fcntl F_NOCACHE enable");
94 for (int size = 0; size < FILE_SIZE;) {
95 T_QUIET;
96 T_ASSERT_GE_INT((int)write(fd, data_buf, IO_SIZE * 16), 0, "write test file");
97 size += (IO_SIZE * 16);
98 }
99 close(fd);
100 sync();
101 }
102
103 static pthread_t
create_thread(uint32_t thread_id,uint32_t priority,bool fixpri,void * (* start_routine)(void *))104 create_thread(uint32_t thread_id, uint32_t priority, bool fixpri,
105 void *(*start_routine)(void *))
106 {
107 int rv;
108 pthread_t new_thread;
109 struct sched_param param = { .sched_priority = (int)priority };
110 pthread_attr_t attr;
111
112 T_ASSERT_POSIX_ZERO(pthread_attr_init(&attr), "pthread_attr_init");
113
114 T_ASSERT_POSIX_ZERO(pthread_attr_setschedparam(&attr, ¶m),
115 "pthread_attr_setschedparam");
116
117 if (fixpri) {
118 T_ASSERT_POSIX_ZERO(pthread_attr_setschedpolicy(&attr, SCHED_RR),
119 "pthread_attr_setschedpolicy");
120 }
121
122 T_ASSERT_POSIX_ZERO(pthread_create(&new_thread, &attr, start_routine,
123 (void*)(uintptr_t)thread_id), "pthread_create");
124
125 T_ASSERT_POSIX_ZERO(pthread_attr_destroy(&attr), "pthread_attr_destroy");
126
127 threads[thread_id].thread = new_thread;
128
129 return new_thread;
130 }
131
132 /* Spin until a specified number of seconds elapses */
133 static void
spin_for_duration(uint32_t seconds)134 spin_for_duration(uint32_t seconds)
135 {
136 uint64_t duration = nanos_to_abs((uint64_t)seconds * NSEC_PER_SEC);
137 uint64_t current_time = mach_absolute_time();
138 uint64_t timeout = duration + current_time;
139
140 uint64_t spin_count = 0;
141
142 while (mach_absolute_time() < timeout && atomic_load_explicit(&keep_going,
143 memory_order_relaxed)) {
144 spin_count++;
145 }
146 }
147
148 static void *
spin_thread(void * arg)149 spin_thread(void *arg)
150 {
151 uint32_t thread_id = (uint32_t) arg;
152 char name[30] = "";
153
154 snprintf(name, sizeof(name), "spin thread %2d", thread_id);
155 pthread_setname_np(name);
156 T_ASSERT_MACH_SUCCESS(semaphore_wait_signal(semaphore, worker_sem),
157 "semaphore_wait_signal");
158 spin_for_duration(SPIN_SECS);
159 return NULL;
160 }
161
162 void
perform_io(dt_stat_time_t stat)163 perform_io(dt_stat_time_t stat)
164 {
165 /* Open the test data file */
166 int test_file_fd = open("/mnt2/test", O_RDONLY);
167 T_WITH_ERRNO;
168 T_ASSERT_POSIX_SUCCESS(test_file_fd, "Open test data file");
169
170 /* Disable caching and read-ahead for the file */
171 T_ASSERT_POSIX_ZERO(fcntl(test_file_fd, F_NOCACHE, 1), "fcntl F_NOCACHE enable");
172 T_ASSERT_POSIX_ZERO(fcntl(test_file_fd, F_RDAHEAD, 0), "fcntl F_RDAHEAD disable");
173
174 uint32_t count = 0;
175 int ret;
176
177 for (int i = 0; i < WARMUP_ITERATIONS; i++) {
178 /* Warmup loop */
179 read(test_file_fd, data_buf, IO_SIZE);
180 }
181
182 do {
183 T_STAT_MEASURE(stat) {
184 ret = read(test_file_fd, data_buf, IO_SIZE);
185 }
186 if (ret == 0) {
187 T_QUIET;
188 T_ASSERT_POSIX_SUCCESS(lseek(test_file_fd, 0, SEEK_SET), "lseek begin");
189 } else if (ret < 0) {
190 T_FAIL("read failure");
191 T_END;
192 }
193 count++;
194 } while (count < IO_COUNT);
195 close(test_file_fd);
196 }
197
198 T_GLOBAL_META(T_META_NAMESPACE("xnu.io"), T_META_TAG_PERF);
199
200 /* Disable the test on MacOS for now */
201 T_DECL(read_perf, "Sequential Uncached Read Performance", T_META_TYPE_PERF, T_META_CHECK_LEAKS(NO), T_META_ASROOT(YES), T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_NOT_ELIGIBLE)
202 {
203 #if !CONFIG_EMBEDDED
204 T_SKIP("Not supported on MacOS");
205 #endif /* !CONFIG_EMBEDDED */
206
207 io_perf_test_io_init();
208 pthread_setname_np("main thread");
209
210 T_ASSERT_MACH_SUCCESS(mach_timebase_info(&timebase_info), "mach_timebase_info");
211
212 dt_stat_time_t seq_noload = dt_stat_time_create("sequential read latency (CPU idle)");
213 perform_io(seq_noload);
214 dt_stat_finalize(seq_noload);
215
216 /*
217 * We create spinner threads for this test so that all other cores are
218 * busy. That way the I/O issue thread has to context switch to the
219 * IOWorkLoop thread and back for the I/O.
220 */
221 T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &semaphore,
222 SYNC_POLICY_FIFO, 0), "semaphore_create");
223
224 T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &worker_sem,
225 SYNC_POLICY_FIFO, 0), "semaphore_create");
226
227 size_t ncpu_size = sizeof(g_numcpus);
228 T_ASSERT_POSIX_SUCCESS(sysctlbyname("hw.ncpu", &g_numcpus, &ncpu_size, NULL, 0),
229 "sysctlbyname(hw.ncpu)");
230
231 T_LOG("hw.ncpu: %d\n", g_numcpus);
232 uint32_t n_spinners = g_numcpus - 1;
233
234 for (uint32_t thread_id = 0; thread_id < n_spinners; thread_id++) {
235 threads[thread_id].thread = create_thread(thread_id, THR_SPINNER_PRI,
236 true, &spin_thread);
237 }
238
239 for (uint32_t thread_id = 0; thread_id < n_spinners; thread_id++) {
240 T_ASSERT_MACH_SUCCESS(semaphore_wait(worker_sem), "semaphore_wait");
241 }
242
243 T_ASSERT_MACH_SUCCESS(semaphore_signal_all(semaphore), "semaphore_signal");
244
245 dt_stat_time_t seq_load = dt_stat_time_create("sequential read latency (Single CPU)");
246 perform_io(seq_load);
247 dt_stat_finalize(seq_load);
248
249 atomic_store_explicit(&keep_going, 0, memory_order_relaxed);
250 for (uint32_t thread_id = 0; thread_id < n_spinners; thread_id++) {
251 T_ASSERT_POSIX_ZERO(pthread_join(threads[thread_id].thread, NULL),
252 "pthread_join %d", thread_id);
253 }
254 }
255