1 /*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 *
31 */
32
33 #ifndef ARM_CPU_DATA
34 #define ARM_CPU_DATA
35
36 #ifdef MACH_KERNEL_PRIVATE
37
38 #include <mach_assert.h>
39 #include <kern/assert.h>
40 #include <kern/kern_types.h>
41 #include <kern/processor.h>
42 #include <pexpert/pexpert.h>
43 #include <arm/thread.h>
44 #include <arm64/proc_reg.h>
45
46 #include <mach/mach_types.h>
47 #include <machine/thread.h>
48
49 __ASSUME_PTR_ABI_SINGLE_BEGIN
50
51 static inline __attribute__((const)) thread_t
current_thread_fast(void)52 current_thread_fast(void)
53 {
54 #if defined(__arm64__)
55 /*
56 * rdar://73762648 clang nowadays insists that this is not constant
57 *
58 * __builtin_arm_rsr64("TPIDR_EL1")
59 *
60 * and ignores the "attribute const", so do it the "dumb" way.
61 */
62 unsigned long result;
63 __asm__ ("mrs %0, TPIDR_EL1" : "=r" (result));
64 return __unsafe_forge_single(thread_t, result);
65 #else
66 // TPIDRPRW
67 return __unsafe_forge_single(thread_t, __builtin_arm_mrc(15, 0, 13, 0, 4));
68 #endif
69 }
70
71 /*
72 * The "volatile" flavor of current_thread() is intended for use by
73 * scheduler code which may need to update the thread pointer in the
74 * course of a context switch. Any call to current_thread() made
75 * prior to the thread pointer update should be safe to optimize away
76 * as it should be consistent with that thread's state to the extent
77 * the compiler can reason about it. Likewise, the context switch
78 * path will eventually result in an arbitrary branch to the new
79 * thread's pc, about which the compiler won't be able to reason.
80 * Thus any compile-time optimization of current_thread() calls made
81 * within the new thread should be safely encapsulated in its
82 * register/stack state. The volatile form therefore exists to cover
83 * the window between the thread pointer update and the branch to
84 * the new pc.
85 */
86 static inline thread_t
current_thread_volatile(void)87 current_thread_volatile(void)
88 {
89 /*
90 * The compiler might decide to treat rsr64 as const (comes and goes),
91 * which can allow it to eliminate redundant calls, which we don't want
92 * here. Thus we use volatile asm. Which gives us control on semantics.
93 *
94 * The mrc used for arm32 should be treated as volatile however.
95 */
96 #if defined(__arm64__)
97 unsigned long result;
98 __asm__ volatile ("mrs %0, TPIDR_EL1" : "=r" (result));
99 return __unsafe_forge_single(thread_t, result);
100 #else
101 // TPIDRPRW
102 return __unsafe_forge_single(thread_t, __builtin_arm_mrc(15, 0, 13, 0, 4));
103 #endif
104 }
105
106 #if defined(__arm64__)
107
108 static inline vm_offset_t
exception_stack_pointer(void)109 exception_stack_pointer(void)
110 {
111 vm_offset_t result = 0;
112 __asm__ volatile (
113 "msr SPSel, #1 \n"
114 "mov %0, sp \n"
115 "msr SPSel, #0 \n"
116 : "=r" (result));
117
118 return result;
119 }
120
121 #endif /* defined(__arm64__) */
122
123 #define getCpuDatap() current_thread()->machine.CpuDatap
124 #define current_cpu_datap() getCpuDatap()
125
126 extern int get_preemption_level(void);
127 extern unsigned int get_preemption_level_for_thread(thread_t);
128
129 #define mp_disable_preemption() _disable_preemption()
130 #define mp_enable_preemption() _enable_preemption()
131
132 __ASSUME_PTR_ABI_SINGLE_END
133
134 #endif /* MACH_KERNEL_PRIVATE */
135
136 #endif /* ARM_CPU_DATA */
137