1 /*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/monotonic.h>
30 #include <kern/recount.h>
31 #include <machine/machine_routines.h>
32 #include <machine/monotonic.h>
33 #include <pexpert/pexpert.h>
34 #include <sys/param.h> /* NULL */
35 #include <sys/stat.h> /* dev_t */
36 #include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
37 #include <sys/conf.h> /* must come after sys/stat.h */
38 #include <sys/resource_private.h>
39 #include <sys/sysctl.h>
40 #include <sys/sysproto.h>
41 #include <sys/systm.h>
42 #include <sys/types.h>
43 #include <sys/monotonic.h>
44 #include <kern/cpc.h>
45
46 static int mt_cdev_open(dev_t dev, int flags, int devtype, proc_t p);
47 static int mt_cdev_close(dev_t dev, int flags, int devtype, proc_t p);
48 static int mt_cdev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag,
49 proc_t p);
50
51 #define MT_NODE "monotonic"
52
53 static const struct cdevsw mt_cdevsw = {
54 .d_open = mt_cdev_open,
55 .d_close = mt_cdev_close,
56 .d_ioctl = mt_cdev_ioctl,
57
58 .d_read = eno_rdwrt, .d_write = eno_rdwrt, .d_stop = eno_stop,
59 .d_reset = eno_reset, .d_ttys = NULL, .d_select = eno_select,
60 .d_mmap = eno_mmap, .d_strategy = eno_strat, .d_type = 0
61 };
62
63 /*
64 * Written at initialization, read-only thereafter.
65 */
66 LCK_GRP_DECLARE(mt_lock_grp, MT_NODE);
67 static int mt_dev_major;
68
69 static mt_device_t
mt_get_device(dev_t devnum)70 mt_get_device(dev_t devnum)
71 {
72 const int minor_dev = minor(devnum);
73 if (minor_dev < 0 || minor_dev >= MT_NDEVS) {
74 return NULL;
75 }
76 return &mt_devices[minor_dev];
77 }
78
79 static void
mt_device_lock(mt_device_t dev)80 mt_device_lock(mt_device_t dev)
81 {
82 lck_mtx_lock(&dev->mtd_lock);
83 }
84
85 static void
mt_device_unlock(mt_device_t dev)86 mt_device_unlock(mt_device_t dev)
87 {
88 lck_mtx_unlock(&dev->mtd_lock);
89 }
90
91 static void
mt_device_assert_lock_held(__assert_only mt_device_t dev)92 mt_device_assert_lock_held(__assert_only mt_device_t dev)
93 {
94 LCK_MTX_ASSERT(&dev->mtd_lock, LCK_MTX_ASSERT_OWNED);
95 }
96
97 static void
mt_device_assert_inuse(__assert_only mt_device_t dev)98 mt_device_assert_inuse(__assert_only mt_device_t dev)
99 {
100 assert(dev->mtd_inuse == true);
101 }
102
103 int
mt_dev_init(void)104 mt_dev_init(void)
105 {
106 mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw);
107 if (mt_dev_major < 0) {
108 panic("monotonic: cdevsw_add failed: %d", mt_dev_major);
109 }
110
111 for (int i = 0; i < MT_NDEVS; i++) {
112 if (mt_devices[i].mtd_init(&mt_devices[i])) {
113 continue;
114 }
115
116 assert(mt_devices[i].mtd_ncounters > 0);
117
118 dev_t dev = makedev(mt_dev_major, i);
119 void *node = devfs_make_node(dev, DEVFS_CHAR, UID_ROOT,
120 GID_WINDOWSERVER, 0666, MT_NODE "/%s",
121 mt_devices[i].mtd_name);
122 if (!node) {
123 panic("monotonic: devfs_make_node failed for '%s'",
124 mt_devices[i].mtd_name);
125 }
126
127 lck_mtx_init(&mt_devices[i].mtd_lock, &mt_lock_grp, LCK_ATTR_NULL);
128 }
129
130 return 0;
131 }
132
133 static int
mt_cdev_open(dev_t devnum,__unused int flags,__unused int devtype,__unused proc_t p)134 mt_cdev_open(dev_t devnum, __unused int flags, __unused int devtype,
135 __unused proc_t p)
136 {
137 int error = 0;
138
139 mt_device_t dev = mt_get_device(devnum);
140 if (!dev) {
141 return ENODEV;
142 }
143 if (!cpc_hw_acquire(CPC_HW_UPMU, "monotonic")) {
144 return EBUSY;
145 }
146 mt_device_lock(dev);
147 if (dev->mtd_inuse) {
148 error = EALREADY;
149 } else if (!mt_acquire_counters()) {
150 error = ECONNREFUSED;
151 } else {
152 dev->mtd_reset();
153 dev->mtd_inuse = true;
154 }
155 mt_device_unlock(dev);
156
157 if (error != 0) {
158 cpc_hw_release(CPC_HW_UPMU, "monotonic");
159 }
160 return error;
161 }
162
163 static int
mt_cdev_close(dev_t devnum,__unused int flags,__unused int devtype,__unused struct proc * p)164 mt_cdev_close(dev_t devnum, __unused int flags, __unused int devtype,
165 __unused struct proc *p)
166 {
167 mt_device_t dev = mt_get_device(devnum);
168 if (!dev) {
169 return ENODEV;
170 }
171
172 cpc_hw_release(CPC_HW_UPMU, "monotonic");
173
174 mt_device_lock(dev);
175 mt_device_assert_inuse(dev);
176 dev->mtd_inuse = false;
177 dev->mtd_reset();
178 mt_release_counters();
179 mt_device_unlock(dev);
180
181 return 0;
182 }
183
184 static int
mt_ctl_add(mt_device_t dev,user_addr_t uptr)185 mt_ctl_add(mt_device_t dev, user_addr_t uptr)
186 {
187 int error;
188 uint32_t ctr;
189 union monotonic_ctl_add ctl;
190
191 mt_device_assert_lock_held(dev);
192
193 error = copyin(uptr, &ctl, sizeof(ctl.in));
194 if (error) {
195 return error;
196 }
197
198 error = dev->mtd_add(&ctl.in.config, &ctr);
199 if (error) {
200 return error;
201 }
202
203 ctl.out.ctr = ctr;
204
205 error = copyout(&ctl, uptr, sizeof(ctl.out));
206 if (error) {
207 return error;
208 }
209
210 return 0;
211 }
212
213 static int
mt_ctl_counts(mt_device_t dev,user_addr_t uptr)214 mt_ctl_counts(mt_device_t dev, user_addr_t uptr)
215 {
216 int error;
217 union monotonic_ctl_counts ctl;
218
219 mt_device_assert_lock_held(dev);
220
221 error = copyin(uptr, &ctl, sizeof(ctl.in));
222 if (error) {
223 return error;
224 }
225
226 if (ctl.in.ctr_mask == 0) {
227 return EINVAL;
228 }
229
230 {
231 uint64_t counts[dev->mtd_nmonitors][dev->mtd_ncounters];
232 memset(counts, 0,
233 dev->mtd_ncounters * dev->mtd_nmonitors * sizeof(counts[0][0]));
234 error = dev->mtd_read(ctl.in.ctr_mask, (uint64_t *)counts);
235 if (error) {
236 return error;
237 }
238
239 error = copyout(&counts, uptr, sizeof(counts));
240 if (error) {
241 return error;
242 }
243 }
244
245 return 0;
246 }
247
248 static int
mt_ctl_enable(mt_device_t dev,user_addr_t uptr)249 mt_ctl_enable(mt_device_t dev, user_addr_t uptr)
250 {
251 int error;
252 union monotonic_ctl_enable ctl;
253
254 mt_device_assert_lock_held(dev);
255
256 error = copyin(uptr, &ctl, sizeof(ctl));
257 if (error) {
258 return error;
259 }
260
261 dev->mtd_enable(ctl.in.enable);
262
263 return 0;
264 }
265
266 static int
mt_ctl_reset(mt_device_t dev)267 mt_ctl_reset(mt_device_t dev)
268 {
269 mt_device_assert_lock_held(dev);
270 dev->mtd_reset();
271 return 0;
272 }
273
274 static int
mt_cdev_ioctl(dev_t devnum,unsigned long cmd,char * arg,__unused int flags,__unused proc_t p)275 mt_cdev_ioctl(dev_t devnum, unsigned long cmd, char *arg, __unused int flags,
276 __unused proc_t p)
277 {
278 int error = ENODEV;
279 user_addr_t uptr = *(user_addr_t *)(void *)arg;
280
281 mt_device_t dev = mt_get_device(devnum);
282 if (!dev) {
283 return ENODEV;
284 }
285 mt_device_lock(dev);
286
287 switch (cmd) {
288 case MT_IOC_RESET:
289 error = mt_ctl_reset(dev);
290 break;
291
292 case MT_IOC_ADD:
293 error = mt_ctl_add(dev, uptr);
294 break;
295
296 case MT_IOC_ENABLE:
297 error = mt_ctl_enable(dev, uptr);
298 break;
299
300 case MT_IOC_COUNTS:
301 error = mt_ctl_counts(dev, uptr);
302 break;
303
304 case MT_IOC_GET_INFO: {
305 union monotonic_ctl_info info = {
306 .out = {
307 .nmonitors = dev->mtd_nmonitors,
308 .ncounters = dev->mtd_ncounters,
309 },
310 };
311 error = copyout(&info, uptr, sizeof(info));
312 break;
313 }
314
315 default:
316 error = ENODEV;
317 break;
318 }
319
320 mt_device_unlock(dev);
321
322 return error;
323 }
324
325 static void
_convert_usage_to_counts(struct recount_usage * usage,uint64_t * counts)326 _convert_usage_to_counts(struct recount_usage *usage, uint64_t *counts)
327 {
328 #if CONFIG_PERVASIVE_CPI
329 counts[MT_CORE_INSTRS] = usage->ru_metrics[RCT_LVL_KERNEL].rm_instructions;
330 counts[MT_CORE_CYCLES] = usage->ru_metrics[RCT_LVL_KERNEL].rm_cycles;
331 #else /* CONFIG_PERVASIVE_CPI */
332 #pragma unused(usage, counts)
333 #endif /* !CONFIG_PERVASIVE_CPI */
334 }
335
336 enum mt_sysctl {
337 MT_SUPPORTED,
338 MT_PMIS,
339 MT_RETROGRADE,
340 MT_TASK_THREAD,
341 MT_DEBUG,
342 MT_KDBG_TEST,
343 MT_FIX_CPU_PERF,
344 MT_FIX_THREAD_PERF,
345 MT_FIX_TASK_PERF,
346 };
347
348 static int
349 mt_sysctl SYSCTL_HANDLER_ARGS
350 {
351 #pragma unused(oidp, arg2)
352 uint64_t start[MT_CORE_NFIXED] = { 0 }, end[MT_CORE_NFIXED] = { 0 };
353 uint64_t counts[2] = { 0 };
354
355 switch ((enum mt_sysctl)arg1) {
356 case MT_SUPPORTED:
357 return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
358 case MT_PMIS:
359 return sysctl_io_number(req, mt_count_pmis(), sizeof(uint64_t), NULL, NULL);
360 case MT_RETROGRADE: {
361 uint64_t value = os_atomic_load_wide(&mt_retrograde, relaxed);
362 return sysctl_io_number(req, value, sizeof(mt_retrograde), NULL, NULL);
363 }
364 case MT_TASK_THREAD:
365 return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
366 case MT_DEBUG: {
367 int value = mt_debug;
368
369 int r = sysctl_io_number(req, value, sizeof(value), &value, NULL);
370 if (r) {
371 return r;
372 }
373 mt_debug = value;
374
375 return 0;
376 }
377 case MT_KDBG_TEST: {
378 if (req->newptr == USER_ADDR_NULL) {
379 return EINVAL;
380 }
381
382 int intrs_en = ml_set_interrupts_enabled(FALSE);
383 MT_KDBG_TMPCPU_START(0x3fff);
384 MT_KDBG_TMPCPU_END(0x3fff);
385 ml_set_interrupts_enabled(intrs_en);
386
387 return 0;
388 }
389 case MT_FIX_CPU_PERF: {
390 int intrs_en = ml_set_interrupts_enabled(FALSE);
391 mt_fixed_counts(start);
392 mt_fixed_counts(end);
393 ml_set_interrupts_enabled(intrs_en);
394 goto copyout_counts;
395 }
396 case MT_FIX_THREAD_PERF: {
397 int intrs_en = ml_set_interrupts_enabled(FALSE);
398 struct recount_usage start_usage = { 0 };
399 struct recount_usage end_usage = { 0 };
400 recount_current_thread_usage(&start_usage);
401 recount_current_thread_usage(&end_usage);
402 ml_set_interrupts_enabled(intrs_en);
403 _convert_usage_to_counts(&start_usage, start);
404 _convert_usage_to_counts(&end_usage, end);
405
406 goto copyout_counts;
407 }
408 case MT_FIX_TASK_PERF: {
409 int intrs_en = ml_set_interrupts_enabled(FALSE);
410 struct recount_usage start_usage = { 0 };
411 struct recount_usage end_usage = { 0 };
412 recount_current_task_usage(&start_usage);
413 recount_current_task_usage(&end_usage);
414 ml_set_interrupts_enabled(intrs_en);
415 _convert_usage_to_counts(&start_usage, start);
416 _convert_usage_to_counts(&end_usage, end);
417
418 goto copyout_counts;
419 }
420 default:
421 return ENOENT;
422 }
423
424 copyout_counts:
425 counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS];
426 counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES];
427
428 return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts)));
429 }
430
431 SYSCTL_DECL(_kern_monotonic);
432 SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
433 "monotonic");
434
435 #define MT_SYSCTL(NAME, ARG, FLAGS, SIZE, SIZESTR, DESC) \
436 SYSCTL_PROC(_kern_monotonic, OID_AUTO, NAME, \
437 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | (FLAGS), \
438 (void *)(ARG), SIZE, mt_sysctl, SIZESTR, DESC)
439
440 MT_SYSCTL(supported, MT_SUPPORTED, 0, sizeof(int), "I",
441 "whether monotonic is supported");
442 MT_SYSCTL(debug, MT_DEBUG, CTLFLAG_MASKED, sizeof(int), "I",
443 "whether monotonic is printing debug messages");
444 MT_SYSCTL(pmis, MT_PMIS, 0, sizeof(uint64_t), "Q",
445 "number of PMIs seen");
446 MT_SYSCTL(retrograde_updates, MT_RETROGRADE, 0, sizeof(uint64_t), "Q",
447 "number of times a counter appeared to go backwards");
448 MT_SYSCTL(task_thread_counting, MT_TASK_THREAD, 0, sizeof(int), "I",
449 "whether task and thread counting is enabled");
450 MT_SYSCTL(kdebug_test, MT_KDBG_TEST, CTLFLAG_MASKED, sizeof(int), "O",
451 "whether task and thread counting is enabled");
452 MT_SYSCTL(fixed_cpu_perf, MT_FIX_CPU_PERF, CTLFLAG_MASKED,
453 sizeof(uint64_t) * 2, "O",
454 "overhead of accessing the current CPU's counters");
455 MT_SYSCTL(fixed_thread_perf, MT_FIX_THREAD_PERF, CTLFLAG_MASKED,
456 sizeof(uint64_t) * 2, "O",
457 "overhead of accessing the current thread's counters");
458 MT_SYSCTL(fixed_task_perf, MT_FIX_TASK_PERF, CTLFLAG_MASKED,
459 sizeof(uint64_t) * 2, "O",
460 "overhead of accessing the current task's counters");
461