1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2001 Jake Burkholder.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD$
29 */
30
31 #ifndef _MACHINE_CPUFUNC_H_
32 #define _MACHINE_CPUFUNC_H_
33
34 #include <machine/asi.h>
35 #include <machine/pstate.h>
36
37 struct thread;
38
39 /*
40 * Membar operand macros for use in other macros when # is a special
41 * character. Keep these in sync with what the hardware expects.
42 */
43 #define C_Lookaside (0)
44 #define C_MemIssue (1)
45 #define C_Sync (2)
46 #define M_LoadLoad (0)
47 #define M_StoreLoad (1)
48 #define M_LoadStore (2)
49 #define M_StoreStore (3)
50
51 #define CMASK_SHIFT (4)
52 #define MMASK_SHIFT (0)
53
54 #define CMASK_GEN(bit) ((1 << (bit)) << CMASK_SHIFT)
55 #define MMASK_GEN(bit) ((1 << (bit)) << MMASK_SHIFT)
56
57 #define Lookaside CMASK_GEN(C_Lookaside)
58 #define MemIssue CMASK_GEN(C_MemIssue)
59 #define Sync CMASK_GEN(C_Sync)
60 #define LoadLoad MMASK_GEN(M_LoadLoad)
61 #define StoreLoad MMASK_GEN(M_StoreLoad)
62 #define LoadStore MMASK_GEN(M_LoadStore)
63 #define StoreStore MMASK_GEN(M_StoreStore)
64
65 #define casa(rs1, rs2, rd, asi) ({ \
66 u_int __rd = (uint32_t)(rd); \
67 __asm __volatile("casa [%2] %3, %4, %0" \
68 : "+r" (__rd), "=m" (*rs1) \
69 : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1)); \
70 __rd; \
71 })
72
73 #define casxa(rs1, rs2, rd, asi) ({ \
74 u_long __rd = (uint64_t)(rd); \
75 __asm __volatile("casxa [%2] %3, %4, %0" \
76 : "+r" (__rd), "=m" (*rs1) \
77 : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1)); \
78 __rd; \
79 })
80
81 #define flush(va) do { \
82 __asm __volatile("flush %0" : : "r" (va)); \
83 } while (0)
84
85 #define flushw() do { \
86 __asm __volatile("flushw" : :); \
87 } while (0)
88
89 #define mov(val, reg) do { \
90 __asm __volatile("mov %0, %" __XSTRING(reg) : : "r" (val)); \
91 } while (0)
92
93 /* Generate ld*a/st*a functions for non-constant ASIs. */
94 #define LDNC_GEN(tp, o) \
95 static __inline tp \
96 o ## _nc(caddr_t va, int asi) \
97 { \
98 tp r; \
99 __asm __volatile("wr %2, 0, %%asi;" #o " [%1] %%asi, %0"\
100 : "=r" (r) : "r" (va), "r" (asi)); \
101 return (r); \
102 }
103
104 LDNC_GEN(u_char, lduba);
105 LDNC_GEN(u_short, lduha);
106 LDNC_GEN(u_int, lduwa);
107 LDNC_GEN(u_long, ldxa);
108
109 #define LD_GENERIC(va, asi, op, type) ({ \
110 type __r; \
111 __asm __volatile(#op " [%1] %2, %0" \
112 : "=r" (__r) : "r" (va), "n" (asi)); \
113 __r; \
114 })
115
116 #define lduba(va, asi) LD_GENERIC(va, asi, lduba, u_char)
117 #define lduha(va, asi) LD_GENERIC(va, asi, lduha, u_short)
118 #define lduwa(va, asi) LD_GENERIC(va, asi, lduwa, u_int)
119 #define ldxa(va, asi) LD_GENERIC(va, asi, ldxa, u_long)
120
121 #define STNC_GEN(tp, o) \
122 static __inline void \
123 o ## _nc(caddr_t va, int asi, tp val) \
124 { \
125 __asm __volatile("wr %2, 0, %%asi;" #o " %0, [%1] %%asi"\
126 : : "r" (val), "r" (va), "r" (asi)); \
127 }
128
129 STNC_GEN(u_char, stba);
130 STNC_GEN(u_short, stha);
131 STNC_GEN(u_int, stwa);
132 STNC_GEN(u_long, stxa);
133
134 #define ST_GENERIC(va, asi, val, op) \
135 __asm __volatile(#op " %0, [%1] %2" \
136 : : "r" (val), "r" (va), "n" (asi)); \
137
138 #define stba(va, asi, val) ST_GENERIC(va, asi, val, stba)
139 #define stha(va, asi, val) ST_GENERIC(va, asi, val, stha)
140 #define stwa(va, asi, val) ST_GENERIC(va, asi, val, stwa)
141 #define stxa(va, asi, val) ST_GENERIC(va, asi, val, stxa)
142
143 /*
144 * Attempt to read from addr, val. If a Data Access Error trap happens,
145 * they return -1 and the contents of val is undefined. A return of 0
146 * means no trap happened, and the contents of val is valid.
147 */
148 int fasword8(u_long asi, void *addr, uint8_t *val);
149 int fasword16(u_long asi, void *addr, uint16_t *val);
150 int fasword32(u_long asi, void *addr, uint32_t *val);
151
152 #define membar(mask) do { \
153 __asm __volatile("membar %0" : : "n" (mask) : "memory"); \
154 } while (0)
155
156 #define rd(name) ({ \
157 uint64_t __sr; \
158 __asm __volatile("rd %%" #name ", %0" : "=r" (__sr) :); \
159 __sr; \
160 })
161
162 #define wr(name, val, xorval) do { \
163 __asm __volatile("wr %0, %1, %%" #name \
164 : : "r" (val), "rI" (xorval)); \
165 } while (0)
166
167 #define rdpr(name) ({ \
168 uint64_t __pr; \
169 __asm __volatile("rdpr %%" #name", %0" : "=r" (__pr) :); \
170 __pr; \
171 })
172
173 #define wrpr(name, val, xorval) do { \
174 __asm __volatile("wrpr %0, %1, %%" #name \
175 : : "r" (val), "rI" (xorval)); \
176 } while (0)
177
178 /*
179 * Trick GAS/GCC into compiling access to TICK/(S)TICK_COMPARE independently
180 * of the selected instruction set.
181 */
182 #define rdtickcmpr() rd(asr23)
183 #define rdstick() rd(asr24)
184 #define rdstickcmpr() rd(asr25)
185 #define wrtickcmpr(val, xorval) wr(asr23, (val), (xorval))
186 #define wrstick(val, xorval) wr(asr24, (val), (xorval))
187 #define wrstickcmpr(val, xorval) wr(asr25, (val), (xorval))
188
189 /*
190 * Macro intended to be used instead of wr(asr23, val, xorval) for writing to
191 * the TICK_COMPARE register in order to avoid a bug in BlackBird CPUs that
192 * can cause these writes to fail under certain conditions which in turn
193 * causes the hardclock to stop. The workaround is to read the TICK_COMPARE
194 * register back immediately after writing to it with these two instructions
195 * aligned to a quadword boundary in order to ensure that I$ misses won't
196 * split them up.
197 */
198 #define wrtickcmpr_bbwar(val, xorval) ({ \
199 __asm __volatile( \
200 " ba,pt %%xcc, 1f ; " \
201 " nop ; " \
202 " .align 128 ; " \
203 "1: wr %0, %1, %%asr23 ; " \
204 " rd %%asr23, %%g0 ; " \
205 : : "r" (val), "rI" (xorval)); \
206 })
207
208 static __inline void
breakpoint(void)209 breakpoint(void)
210 {
211
212 __asm __volatile("ta %%xcc, 1" : :);
213 }
214
215 static __inline register_t
intr_disable(void)216 intr_disable(void)
217 {
218 register_t s;
219
220 s = rdpr(pstate);
221 wrpr(pstate, s & ~PSTATE_IE, 0);
222 return (s);
223 }
224 #define intr_restore(s) wrpr(pstate, (s), 0)
225
226 /*
227 * In some places, it is required that the store is directly followed by a
228 * membar #Sync. Don't trust the compiler to not insert instructions in
229 * between. We also need to disable interrupts completely.
230 */
231 #define stxa_sync(va, asi, val) do { \
232 register_t s; \
233 s = intr_disable(); \
234 __asm __volatile("stxa %0, [%1] %2; membar #Sync" \
235 : : "r" (val), "r" (va), "n" (asi)); \
236 intr_restore(s); \
237 } while (0)
238
239 void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len);
240 void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len);
241 void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len);
242 void aszero(u_long asi, vm_offset_t dst, size_t len);
243
244 /*
245 * Ultrasparc II doesn't implement popc in hardware.
246 */
247 #if 0
248 #define HAVE_INLINE_FFS
249 /*
250 * See page 202 of the SPARC v9 Architecture Manual.
251 */
252 static __inline int
253 ffs(int mask)
254 {
255 int result;
256 int neg;
257 int tmp;
258
259 __asm __volatile(
260 " neg %3, %1 ; "
261 " xnor %3, %1, %2 ; "
262 " popc %2, %0 ; "
263 " movrz %3, %%g0, %0 ; "
264 : "=r" (result), "=r" (neg), "=r" (tmp) : "r" (mask));
265 return (result);
266 }
267 #endif
268
269 #undef LDNC_GEN
270 #undef STNC_GEN
271
272 #endif /* !_MACHINE_CPUFUNC_H_ */
273