1 /*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef OS_TSD_H
30 #define OS_TSD_H
31
32 /* The low nine slots of the TSD are reserved for libsyscall usage. */
33 #define __TSD_RESERVED_BASE 0
34 #define __TSD_RESERVED_MAX 9
35
36 #define __TSD_THREAD_SELF 0
37 #define __TSD_ERRNO 1
38 #define __TSD_MIG_REPLY 2
39 #define __TSD_MACH_THREAD_SELF 3
40 #define __TSD_THREAD_QOS_CLASS 4
41 #define __TSD_RETURN_TO_KERNEL 5
42 /* slot 6 is reserved for Windows/WINE compatibility reasons */
43 #define __TSD_PTR_MUNGE 7
44 #define __TSD_MACH_SPECIAL_REPLY 8
45 #define __TSD_SEMAPHORE_CACHE 9
46
47 #define __TSD_MACH_MSG_AUX 123
48
49
50 #define __TPIDR_CPU_NUM_SHIFT 0
51 #define __TPIDR_CPU_NUM_MASK 0x0000000000000fff
52 #define __TPIDR_CPU_CLUSTER_ID_SHIFT 12
53 #define __TPIDR_CPU_CLUSTER_ID_MASK 0x00000000000ff000
54
55 #ifndef __ASSEMBLER__
56
57 #include <stdint.h>
58 #include <TargetConditionals.h>
59
60 #ifdef __arm__
61 #include <arm/arch.h>
62 #endif
63
64 extern void _thread_set_tsd_base(void *tsd_base);
65
66 /*
67 * The implementation details of this function are not ABI and are subject to change,
68 * do not copy them in another project
69 */
70 __attribute__((always_inline))
71 static __inline__ unsigned int
_os_cpu_number(void)72 _os_cpu_number(void)
73 {
74 #if defined(__arm__)
75 uintptr_t p;
76 __asm__ __volatile__ ("mrc p15, 0, %[p], c13, c0, 3" : [p] "=&r" (p));
77 return (unsigned int)(p & 0x3ul);
78 #elif defined(__arm64__)
79 uint64_t p;
80 __asm__ __volatile__ ("mrs %0, TPIDR_EL0" : "=r" (p));
81 return (p & __TPIDR_CPU_NUM_MASK) >> __TPIDR_CPU_NUM_SHIFT;
82 #elif defined(__x86_64__) || defined(__i386__)
83 struct { uintptr_t p1, p2; } p;
84 __asm__ __volatile__ ("sidt %[p]" : [p] "=&m" (p));
85 return (unsigned int)(p.p1 & 0xfff);
86 #else
87 #error _os_cpu_number not implemented on this architecture
88 #endif
89 }
90
91 /*
92 * The implementation details of this function are not ABI and are subject to change,
93 * do not copy them in another project
94 */
95 __attribute__((always_inline))
96 static __inline__ unsigned int
_os_cpu_cluster_number(void)97 _os_cpu_cluster_number(void)
98 {
99 #if defined(__arm64__)
100 uint64_t p;
101 __asm__ __volatile__ ("mrs %0, TPIDR_EL0" : "=r" (p));
102 return (p & __TPIDR_CPU_CLUSTER_ID_MASK) >> __TPIDR_CPU_CLUSTER_ID_SHIFT;
103 #elif defined(__arm__) || defined(__x86_64__) || defined(__i386__)
104 return 0;
105 #else
106 #error _os_cpu_cluster_number not implemented on this architecture
107 #endif
108 }
109
110 #if defined(__i386__) || defined(__x86_64__)
111
112 #if defined(__has_attribute)
113 #if __has_attribute(address_space)
114 #define OS_GS_RELATIVE __attribute__((address_space(256)))
115 #endif
116 #endif
117
118 #ifdef OS_GS_RELATIVE
119 #define _os_tsd_get_base() ((void * OS_GS_RELATIVE *)0)
120 #else
121 __attribute__((always_inline))
122 static __inline__ void*
_os_tsd_get_direct(unsigned long slot)123 _os_tsd_get_direct(unsigned long slot)
124 {
125 void *ret;
126 __asm__("mov %%gs:%1, %0" : "=r" (ret) : "m" (*(void **)(slot * sizeof(void *))));
127 return ret;
128 }
129
130 __attribute__((always_inline))
131 static __inline__ int
_os_tsd_set_direct(unsigned long slot,void * val)132 _os_tsd_set_direct(unsigned long slot, void *val)
133 {
134 #if defined(__i386__) && defined(__PIC__)
135 __asm__("movl %1, %%gs:%0" : "=m" (*(void **)(slot * sizeof(void *))) : "rn" (val));
136 #elif defined(__i386__) && !defined(__PIC__)
137 __asm__("movl %1, %%gs:%0" : "=m" (*(void **)(slot * sizeof(void *))) : "ri" (val));
138 #else
139 __asm__("movq %1, %%gs:%0" : "=m" (*(void **)(slot * sizeof(void *))) : "rn" (val));
140 #endif
141 return 0;
142 }
143 #endif
144
145 #elif defined(__arm__) || defined(__arm64__)
146
147 __attribute__((always_inline, const))
148 static __inline__ void**
_os_tsd_get_base(void)149 _os_tsd_get_base(void)
150 {
151 #if defined(__arm__)
152 uintptr_t tsd;
153 __asm__("mrc p15, 0, %0, c13, c0, 3\n"
154 "bic %0, %0, #0x3\n" : "=r" (tsd));
155 /* lower 2-bits contain CPU number */
156 #elif defined(__arm64__)
157 /*
158 * <rdar://73762648> Do not use __builtin_arm_rsr64("TPIDRRO_EL0")
159 * so that the "const" attribute takes effect and repeated use
160 * is coalesced properly.
161 */
162 uint64_t tsd;
163 __asm__ ("mrs %0, TPIDRRO_EL0" : "=r" (tsd));
164 #endif
165
166 return (void**)(uintptr_t)tsd;
167 }
168 #define _os_tsd_get_base() _os_tsd_get_base()
169
170 #else
171 #error _os_tsd_get_base not implemented on this architecture
172 #endif
173
174 #ifdef _os_tsd_get_base
175 __attribute__((always_inline))
176 static __inline__ void*
_os_tsd_get_direct(unsigned long slot)177 _os_tsd_get_direct(unsigned long slot)
178 {
179 return _os_tsd_get_base()[slot];
180 }
181
182 __attribute__((always_inline))
183 static __inline__ int
_os_tsd_set_direct(unsigned long slot,void * val)184 _os_tsd_set_direct(unsigned long slot, void *val)
185 {
186 _os_tsd_get_base()[slot] = val;
187 return 0;
188 }
189 #endif
190
191 __attribute__((always_inline, const))
192 static __inline__ uintptr_t
_os_ptr_munge_token(void)193 _os_ptr_munge_token(void)
194 {
195 return (uintptr_t)_os_tsd_get_direct(__TSD_PTR_MUNGE);
196 }
197
198 __attribute__((always_inline, const))
199 static __inline__ uintptr_t
_os_ptr_munge(uintptr_t ptr)200 _os_ptr_munge(uintptr_t ptr)
201 {
202 return ptr ^ _os_ptr_munge_token();
203 }
204 #define _OS_PTR_MUNGE(_ptr) _os_ptr_munge((uintptr_t)(_ptr))
205 #define _OS_PTR_UNMUNGE(_ptr) _os_ptr_munge((uintptr_t)(_ptr))
206
207 #else // __ASSEMBLER__
208
209 #define _OS_TSD_OFFSET(_key) \
210 ((__POINTER_WIDTH__/__CHAR_BIT__)*_key)
211
212 #if defined(__i386__) || defined(__x86_64__)
213
214 #define _OS_PTR_MUNGE(_reg) \
215 xor %gs:_OS_TSD_OFFSET(__TSD_PTR_MUNGE), _reg
216
217 #define _OS_PTR_UNMUNGE(_reg) \
218 _OS_PTR_MUNGE(_reg)
219
220 #elif defined(__arm__) || defined(__arm64__)
221
222 #if defined(__arm__)
223
224 #define _OS_PTR_MUNGE_TOKEN(_reg, _token) \
225 mrc p15, 0, _reg, c13, c0, 3; \
226 bic _reg, _reg, #3; \
227 ldr _token, [ _reg, #_OS_TSD_OFFSET(__TSD_PTR_MUNGE) ]
228
229 #elif defined(__arm64__)
230
231 #define _OS_PTR_MUNGE_TOKEN(_reg, _token) \
232 mrs _reg, TPIDRRO_EL0 %% \
233 ldr _token, [ _reg, #_OS_TSD_OFFSET(__TSD_PTR_MUNGE) ]
234
235 #endif // defined(__arm64__)
236
237 #define _OS_PTR_MUNGE(_regdest, _regsrc, _token) \
238 eor _regdest, _regsrc, _token
239
240 #define _OS_PTR_UNMUNGE(_regdest, _regsrc, _token) \
241 _OS_PTR_MUNGE(_regdest, _regsrc, _token)
242
243 #endif // defined(__arm__) || defined(__arm64__)
244
245 #endif // __ASSEMBLER__
246
247 #endif // OS_TSD_H
248