1 #include <stdio.h>
2 #include <signal.h>
3 #include <sys/sysctl.h>
4 #include <sys/kern_memorystatus.h>
5 #include <mach-o/dyld.h>
6 #include <perfcheck_keys.h>
7
8 #ifdef T_NAMESPACE
9 #undef T_NAMESPACE
10 #endif
11 #include <darwintest.h>
12 #include <darwintest_utils.h>
13
14 T_GLOBAL_META(
15 T_META_NAMESPACE("xnu.vm.perf"),
16 T_META_RADAR_COMPONENT_NAME("xnu"),
17 T_META_RADAR_COMPONENT_VERSION("VM"),
18 T_META_CHECK_LEAKS(false),
19 T_META_TAG_PERF,
20 T_META_ENABLED(false) /* rdar://84443533 */
21 );
22
23 enum {
24 ALL_ZEROS,
25 MOSTLY_ZEROS,
26 RANDOM,
27 TYPICAL
28 };
29
30 #define CREATE_LIST(X) \
31 X(SUCCESS) \
32 X(TOO_FEW_ARGUMENTS) \
33 X(SYSCTL_VM_PAGESIZE_FAILED) \
34 X(VM_PAGESIZE_IS_ZERO) \
35 X(UNKNOWN_PAGE_TYPE) \
36 X(DISPATCH_SOURCE_CREATE_FAILED) \
37 X(INITIAL_SIGNAL_TO_PARENT_FAILED) \
38 X(SIGNAL_TO_PARENT_FAILED) \
39 X(MEMORYSTATUS_CONTROL_FAILED) \
40 X(IS_FREEZABLE_NOT_AS_EXPECTED) \
41 X(EXIT_CODE_MAX)
42
43 #define EXIT_CODES_ENUM(VAR) VAR,
44 enum exit_codes_num {
45 CREATE_LIST(EXIT_CODES_ENUM)
46 };
47
48 #define EXIT_CODES_STRING(VAR) #VAR,
49 static const char *exit_codes_str[] = {
50 CREATE_LIST(EXIT_CODES_STRING)
51 };
52
53 #define SYSCTL_FREEZE_TO_MEMORY "kern.memorystatus_freeze_to_memory=1"
54
55 static pid_t pid = -1;
56 static dt_stat_t ratio;
57 static dt_stat_time_t compr_time;
58 static dt_stat_time_t decompr_time;
59
60 void allocate_zero_pages(char **buf, int num_pages, int vmpgsize);
61 void allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize);
62 void allocate_random_pages(char **buf, int num_pages, int vmpgsize);
63 void allocate_representative_pages(char **buf, int num_pages, int vmpgsize);
64 void run_compressor_test(int size_mb, int page_type);
65 void freeze_helper_process(void);
66 void cleanup(void);
67
68 void
allocate_zero_pages(char ** buf,int num_pages,int vmpgsize)69 allocate_zero_pages(char **buf, int num_pages, int vmpgsize)
70 {
71 int i;
72
73 for (i = 0; i < num_pages; i++) {
74 buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
75 memset(buf[i], 0, vmpgsize);
76 }
77 }
78
79 void
allocate_mostly_zero_pages(char ** buf,int num_pages,int vmpgsize)80 allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize)
81 {
82 int i, j;
83
84 for (i = 0; i < num_pages; i++) {
85 buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
86 memset(buf[i], 0, vmpgsize);
87 for (j = 0; j < 40; j++) {
88 buf[i][j] = (char)(j + 1);
89 }
90 }
91 }
92
93 void
allocate_random_pages(char ** buf,int num_pages,int vmpgsize)94 allocate_random_pages(char **buf, int num_pages, int vmpgsize)
95 {
96 int i;
97
98 for (i = 0; i < num_pages; i++) {
99 buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
100 arc4random_buf((void*)buf[i], (size_t)vmpgsize);
101 }
102 }
103
104 // Gives us the compression ratio we see in the typical case (~2.7)
105 void
allocate_representative_pages(char ** buf,int num_pages,int vmpgsize)106 allocate_representative_pages(char **buf, int num_pages, int vmpgsize)
107 {
108 int i, j;
109 char val;
110
111 for (j = 0; j < num_pages; j++) {
112 buf[j] = (char*)malloc((size_t)vmpgsize * sizeof(char));
113 val = 0;
114 for (i = 0; i < vmpgsize; i += 16) {
115 memset(&buf[j][i], val, 16);
116 if (i < 3400 * (vmpgsize / 4096)) {
117 val++;
118 }
119 }
120 }
121 }
122
123 void
freeze_helper_process(void)124 freeze_helper_process(void)
125 {
126 int ret, freeze_enabled;
127 int64_t compressed_before, compressed_after, input_before, input_after;
128 size_t length;
129 int errno_sysctl_freeze;
130
131 length = sizeof(compressed_before);
132 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_before, &length, NULL, 0),
133 "failed to query vm.compressor_compressed_bytes");
134 length = sizeof(input_before);
135 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_before, &length, NULL, 0),
136 "failed to query vm.compressor_input_bytes");
137
138 T_STAT_MEASURE(compr_time) {
139 ret = sysctlbyname("kern.memorystatus_freeze", NULL, NULL, &pid, sizeof(pid));
140 errno_sysctl_freeze = errno;
141 };
142
143 length = sizeof(compressed_after);
144 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_after, &length, NULL, 0),
145 "failed to query vm.compressor_compressed_bytes");
146 length = sizeof(input_after);
147 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_after, &length, NULL, 0),
148 "failed to query vm.compressor_input_bytes");
149
150 length = sizeof(freeze_enabled);
151 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
152 "failed to query vm.freeze_enabled");
153 if (freeze_enabled) {
154 errno = errno_sysctl_freeze;
155 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_freeze failed");
156 } else {
157 /* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
158 T_LOG("Freeze has been disabled. Terminating early.");
159 T_END;
160 }
161
162 dt_stat_add(ratio, (double)(input_after - input_before) / (double)(compressed_after - compressed_before));
163
164 ret = sysctlbyname("kern.memorystatus_thaw", NULL, NULL, &pid, sizeof(pid));
165 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_thaw failed");
166
167 T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGUSR1), "failed to send SIGUSR1 to child process");
168 }
169
170 void
cleanup(void)171 cleanup(void)
172 {
173 /* No helper process. */
174 if (pid == -1) {
175 return;
176 }
177 /* Kill the helper process. */
178 kill(pid, SIGKILL);
179 }
180
181 void
run_compressor_test(int size_mb,int page_type)182 run_compressor_test(int size_mb, int page_type)
183 {
184 int ret;
185 char sz_str[50];
186 char pt_str[50];
187 char **launch_tool_args;
188 char testpath[PATH_MAX];
189 uint32_t testpath_buf_size;
190 dispatch_source_t ds_freeze, ds_proc, ds_decompr;
191 int freeze_enabled;
192 size_t length;
193 __block bool decompr_latency_is_stable = false;
194
195 length = sizeof(freeze_enabled);
196 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
197 "failed to query vm.freeze_enabled");
198 if (!freeze_enabled) {
199 /* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
200 T_SKIP("Freeze has been disabled. Skipping test.");
201 }
202
203 T_ATEND(cleanup);
204
205 ratio = dt_stat_create("(input bytes / compressed bytes)", "compression_ratio");
206 compr_time = dt_stat_time_create("compressor_latency");
207
208 // This sets the A/B failure threshold at 50% of baseline for compressor_latency
209 dt_stat_set_variable((struct dt_stat *)compr_time, kPCFailureThresholdPctVar, 50.0);
210
211 signal(SIGUSR2, SIG_IGN);
212 ds_decompr = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR2, 0, dispatch_get_main_queue());
213 T_QUIET; T_ASSERT_NOTNULL(ds_decompr, "dispatch_source_create (ds_decompr)");
214
215 dispatch_source_set_event_handler(ds_decompr, ^{
216 decompr_latency_is_stable = true;
217 });
218 dispatch_activate(ds_decompr);
219
220 signal(SIGUSR1, SIG_IGN);
221 ds_freeze = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
222 T_QUIET; T_ASSERT_NOTNULL(ds_freeze, "dispatch_source_create (ds_freeze)");
223
224 dispatch_source_set_event_handler(ds_freeze, ^{
225 if (!(dt_stat_stable(compr_time) && decompr_latency_is_stable)) {
226 freeze_helper_process();
227 } else {
228 dt_stat_finalize(compr_time);
229 dt_stat_finalize(ratio);
230
231 kill(pid, SIGKILL);
232 dispatch_source_cancel(ds_freeze);
233 dispatch_source_cancel(ds_decompr);
234 }
235 });
236 dispatch_activate(ds_freeze);
237
238 testpath_buf_size = sizeof(testpath);
239 ret = _NSGetExecutablePath(testpath, &testpath_buf_size);
240 T_QUIET; T_ASSERT_POSIX_ZERO(ret, "_NSGetExecutablePath");
241 T_LOG("Executable path: %s", testpath);
242
243 sprintf(sz_str, "%d", size_mb);
244 sprintf(pt_str, "%d", page_type);
245 launch_tool_args = (char *[]){
246 testpath,
247 "-n",
248 "allocate_pages",
249 "--",
250 sz_str,
251 pt_str,
252 NULL
253 };
254
255 /* Spawn the child process. Suspend after launch until the exit proc handler has been set up. */
256 ret = dt_launch_tool(&pid, launch_tool_args, true, NULL, NULL);
257 if (ret != 0) {
258 T_LOG("dt_launch tool returned %d with error code %d", ret, errno);
259 }
260 T_QUIET; T_ASSERT_POSIX_SUCCESS(pid, "dt_launch_tool");
261
262 ds_proc = dispatch_source_create(DISPATCH_SOURCE_TYPE_PROC, (uintptr_t)pid, DISPATCH_PROC_EXIT, dispatch_get_main_queue());
263 T_QUIET; T_ASSERT_NOTNULL(ds_proc, "dispatch_source_create (ds_proc)");
264
265 dispatch_source_set_event_handler(ds_proc, ^{
266 int status = 0, code = 0;
267 pid_t rc = waitpid(pid, &status, 0);
268 T_QUIET; T_ASSERT_EQ(rc, pid, "waitpid");
269 code = WEXITSTATUS(status);
270
271 if (code == 0) {
272 T_END;
273 } else if (code > 0 && code < EXIT_CODE_MAX) {
274 T_ASSERT_FAIL("Child exited with %s", exit_codes_str[code]);
275 } else {
276 T_ASSERT_FAIL("Child exited with unknown exit code %d", code);
277 }
278 });
279 dispatch_activate(ds_proc);
280
281 T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGCONT), "failed to send SIGCONT to child process");
282 dispatch_main();
283 }
284
285 T_HELPER_DECL(allocate_pages, "allocates pages to compress") {
286 int i, j, ret, size_mb, page_type, vmpgsize, freezable_state;
287 size_t vmpgsize_length;
288 __block int num_pages;
289 __block char **buf;
290 dispatch_source_t ds_signal;
291
292 vmpgsize_length = sizeof(vmpgsize);
293 ret = sysctlbyname("vm.pagesize", &vmpgsize, &vmpgsize_length, NULL, 0);
294 if (ret != 0) {
295 exit(SYSCTL_VM_PAGESIZE_FAILED);
296 }
297 if (vmpgsize == 0) {
298 exit(VM_PAGESIZE_IS_ZERO);
299 }
300
301 if (argc < 2) {
302 exit(TOO_FEW_ARGUMENTS);
303 }
304
305 size_mb = atoi(argv[0]);
306 page_type = atoi(argv[1]);
307 num_pages = size_mb * 1024 * 1024 / vmpgsize;
308 buf = (char**)malloc(sizeof(char*) * (size_t)num_pages);
309
310 // Switch on the type of page requested
311 switch (page_type) {
312 case ALL_ZEROS:
313 allocate_zero_pages(buf, num_pages, vmpgsize);
314 break;
315 case MOSTLY_ZEROS:
316 allocate_mostly_zero_pages(buf, num_pages, vmpgsize);
317 break;
318 case RANDOM:
319 allocate_random_pages(buf, num_pages, vmpgsize);
320 break;
321 case TYPICAL:
322 allocate_representative_pages(buf, num_pages, vmpgsize);
323 break;
324 default:
325 exit(UNKNOWN_PAGE_TYPE);
326 }
327
328 for (j = 0; j < num_pages; j++) {
329 i = buf[j][0];
330 }
331
332 decompr_time = dt_stat_time_create("decompression_latency");
333
334 /* Opt in to freezing. */
335 printf("[%d] Setting state to freezable\n", getpid());
336 if (memorystatus_control(MEMORYSTATUS_CMD_SET_PROCESS_IS_FREEZABLE, getpid(), 1, NULL, 0) != KERN_SUCCESS) {
337 exit(MEMORYSTATUS_CONTROL_FAILED);
338 }
339
340 /* Verify that the state has been set correctly */
341 freezable_state = memorystatus_control(MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE, getpid(), 0, NULL, 0);
342 if (freezable_state != 1) {
343 exit(IS_FREEZABLE_NOT_AS_EXPECTED);
344 }
345
346 dispatch_after(dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC), dispatch_get_main_queue(), ^{
347 /* Signal to the parent that we're done allocating and it's ok to freeze us */
348 printf("[%d] Sending initial signal to parent to begin freezing\n", getpid());
349 if (kill(getppid(), SIGUSR1) != 0) {
350 exit(INITIAL_SIGNAL_TO_PARENT_FAILED);
351 }
352 });
353
354 signal(SIGUSR1, SIG_IGN);
355 ds_signal = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
356 if (ds_signal == NULL) {
357 exit(DISPATCH_SOURCE_CREATE_FAILED);
358 }
359
360 __block bool collect_dt_stat_measurements = true;
361
362 dispatch_source_set_event_handler(ds_signal, ^{
363 volatile int tmp;
364 uint64_t decompr_start_time, decompr_end_time;
365
366 decompr_start_time = mach_absolute_time();
367
368 /* Make sure all the pages are accessed before trying to freeze again */
369 for (int x = 0; x < num_pages; x++) {
370 tmp = buf[x][0];
371 }
372
373 decompr_end_time = mach_absolute_time();
374
375 if (collect_dt_stat_measurements) {
376 if (dt_stat_stable(decompr_time)) {
377 collect_dt_stat_measurements = false;
378 dt_stat_finalize(decompr_time);
379 if (kill(getppid(), SIGUSR2) != 0) {
380 exit(SIGNAL_TO_PARENT_FAILED);
381 }
382 } else {
383 dt_stat_mach_time_add(decompr_time, decompr_end_time - decompr_start_time);
384 }
385 }
386
387 if (kill(getppid(), SIGUSR1) != 0) {
388 exit(SIGNAL_TO_PARENT_FAILED);
389 }
390 });
391 dispatch_activate(ds_signal);
392
393 dispatch_main();
394 }
395
396 // Numbers for 10MB and above are fairly reproducible. Anything smaller shows a lot of variation.
397
398 // Keeping just the 100MB version for iOSMark
399 #ifndef DT_IOSMARK
400 T_DECL(compr_10MB_zero,
401 "Compression latency for 10MB - zero pages",
402 T_META_ASROOT(true),
403 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY), T_META_TAG_VM_NOT_ELIGIBLE) {
404 run_compressor_test(10, ALL_ZEROS);
405 }
406
407 T_DECL(compr_10MB_mostly_zero,
408 "Compression latency for 10MB - mostly zero pages",
409 T_META_ASROOT(true),
410 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY), T_META_TAG_VM_NOT_ELIGIBLE) {
411 run_compressor_test(10, MOSTLY_ZEROS);
412 }
413
414 T_DECL(compr_10MB_random,
415 "Compression latency for 10MB - random pages",
416 T_META_ASROOT(true),
417 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY), T_META_TAG_VM_NOT_ELIGIBLE) {
418 run_compressor_test(10, RANDOM);
419 }
420
421 T_DECL(compr_10MB_typical,
422 "Compression latency for 10MB - typical pages",
423 T_META_ASROOT(true),
424 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY), T_META_TAG_VM_NOT_ELIGIBLE) {
425 run_compressor_test(10, TYPICAL);
426 }
427
428 T_DECL(compr_100MB_zero,
429 "Compression latency for 100MB - zero pages",
430 T_META_ASROOT(true),
431 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY), T_META_TAG_VM_NOT_ELIGIBLE) {
432 run_compressor_test(100, ALL_ZEROS);
433 }
434
435 T_DECL(compr_100MB_mostly_zero,
436 "Compression latency for 100MB - mostly zero pages",
437 T_META_ASROOT(true),
438 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY), T_META_TAG_VM_NOT_ELIGIBLE) {
439 run_compressor_test(100, MOSTLY_ZEROS);
440 }
441
442 T_DECL(compr_100MB_random,
443 "Compression latency for 100MB - random pages",
444 T_META_ASROOT(true),
445 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY), T_META_TAG_VM_NOT_ELIGIBLE) {
446 run_compressor_test(100, RANDOM);
447 }
448 #endif
449
450 T_DECL(compr_100MB_typical,
451 "Compression latency for 100MB - typical pages",
452 T_META_ASROOT(true),
453 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY), T_META_TAG_VM_NOT_ELIGIBLE) {
454 run_compressor_test(100, TYPICAL);
455 }
456