1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2004-2010 Juli Mallett <[email protected]>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD$
29 */
30
31 #include "opt_ddb.h"
32
33 #include <sys/param.h>
34 #include <sys/kernel.h>
35 #include <sys/systm.h>
36 #include <sys/pcpu.h>
37 #include <sys/smp.h>
38
39 #include <vm/vm.h>
40 #include <vm/pmap.h>
41
42 #include <machine/pte.h>
43 #include <machine/tlb.h>
44
45 #if defined(CPU_CNMIPS)
46 #define MIPS_MAX_TLB_ENTRIES 128
47 #elif defined(CPU_NLM)
48 #define MIPS_MAX_TLB_ENTRIES (2048 + 128)
49 #else
50 #define MIPS_MAX_TLB_ENTRIES 64
51 #endif
52
53 struct tlb_state {
54 unsigned wired;
55 struct tlb_entry {
56 register_t entryhi;
57 register_t entrylo0;
58 register_t entrylo1;
59 register_t pagemask;
60 } entry[MIPS_MAX_TLB_ENTRIES];
61 };
62
63 static struct tlb_state tlb_state[MAXCPU];
64
65 #if 0
66 /*
67 * PageMask must increment in steps of 2 bits.
68 */
69 COMPILE_TIME_ASSERT(POPCNT(TLBMASK_MASK) % 2 == 0);
70 #endif
71
72 static inline void
tlb_probe(void)73 tlb_probe(void)
74 {
75 __asm __volatile ("tlbp" : : : "memory");
76 mips_cp0_sync();
77 }
78
79 static inline void
tlb_read(void)80 tlb_read(void)
81 {
82 __asm __volatile ("tlbr" : : : "memory");
83 mips_cp0_sync();
84 }
85
86 static inline void
tlb_write_indexed(void)87 tlb_write_indexed(void)
88 {
89 __asm __volatile ("tlbwi" : : : "memory");
90 mips_cp0_sync();
91 }
92
93 static void tlb_invalidate_one(unsigned);
94
95 void
tlb_insert_wired(unsigned i,vm_offset_t va,pt_entry_t pte0,pt_entry_t pte1)96 tlb_insert_wired(unsigned i, vm_offset_t va, pt_entry_t pte0, pt_entry_t pte1)
97 {
98 register_t asid;
99 register_t s;
100
101 va &= ~PAGE_MASK;
102
103 s = intr_disable();
104 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
105
106 mips_wr_index(i);
107 mips_wr_pagemask(0);
108 mips_wr_entryhi(TLBHI_ENTRY(va, 0));
109 mips_wr_entrylo0(pte0);
110 mips_wr_entrylo1(pte1);
111 tlb_write_indexed();
112
113 mips_wr_entryhi(asid);
114 intr_restore(s);
115 }
116
117 void
tlb_invalidate_address(struct pmap * pmap,vm_offset_t va)118 tlb_invalidate_address(struct pmap *pmap, vm_offset_t va)
119 {
120 register_t asid;
121 register_t s;
122 int i;
123
124 va &= ~PAGE_MASK;
125
126 s = intr_disable();
127 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
128
129 mips_wr_pagemask(0);
130 mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
131 tlb_probe();
132 i = mips_rd_index();
133 if (i >= 0)
134 tlb_invalidate_one(i);
135
136 mips_wr_entryhi(asid);
137 intr_restore(s);
138 }
139
140 void
tlb_invalidate_all(void)141 tlb_invalidate_all(void)
142 {
143 register_t asid;
144 register_t s;
145 unsigned i;
146
147 s = intr_disable();
148 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
149
150 for (i = mips_rd_wired(); i < num_tlbentries; i++)
151 tlb_invalidate_one(i);
152
153 mips_wr_entryhi(asid);
154 intr_restore(s);
155 }
156
157 void
tlb_invalidate_all_user(struct pmap * pmap)158 tlb_invalidate_all_user(struct pmap *pmap)
159 {
160 register_t asid;
161 register_t s;
162 unsigned i;
163
164 s = intr_disable();
165 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
166
167 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
168 register_t uasid;
169
170 mips_wr_index(i);
171 tlb_read();
172
173 uasid = mips_rd_entryhi() & TLBHI_ASID_MASK;
174 if (pmap == NULL) {
175 /*
176 * Invalidate all non-kernel entries.
177 */
178 if (uasid == 0)
179 continue;
180 } else {
181 /*
182 * Invalidate this pmap's entries.
183 */
184 if (uasid != pmap_asid(pmap))
185 continue;
186 }
187 tlb_invalidate_one(i);
188 }
189
190 mips_wr_entryhi(asid);
191 intr_restore(s);
192 }
193
194 /*
195 * Invalidates any TLB entries that map a virtual page from the specified
196 * address range. If "end" is zero, then every virtual page is considered to
197 * be within the address range's upper bound.
198 */
199 void
tlb_invalidate_range(pmap_t pmap,vm_offset_t start,vm_offset_t end)200 tlb_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
201 {
202 register_t asid, end_hi, hi, hi_pagemask, s, save_asid, start_hi;
203 int i;
204
205 KASSERT(start < end || (end == 0 && start > 0),
206 ("tlb_invalidate_range: invalid range"));
207
208 /*
209 * Truncate the virtual address "start" to an even page frame number,
210 * and round the virtual address "end" to an even page frame number.
211 */
212 start &= ~((1 << TLBMASK_SHIFT) - 1);
213 end = roundup2(end, 1 << TLBMASK_SHIFT);
214
215 s = intr_disable();
216 save_asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
217
218 asid = pmap_asid(pmap);
219 start_hi = TLBHI_ENTRY(start, asid);
220 end_hi = TLBHI_ENTRY(end, asid);
221
222 /*
223 * Select the fastest method for invalidating the TLB entries.
224 */
225 if (end - start < num_tlbentries << TLBMASK_SHIFT || (end == 0 &&
226 start >= -(num_tlbentries << TLBMASK_SHIFT))) {
227 /*
228 * The virtual address range is small compared to the size of
229 * the TLB. Probe the TLB for each even numbered page frame
230 * within the virtual address range.
231 */
232 for (hi = start_hi; hi != end_hi; hi += 1 << TLBMASK_SHIFT) {
233 mips_wr_pagemask(0);
234 mips_wr_entryhi(hi);
235 tlb_probe();
236 i = mips_rd_index();
237 if (i >= 0)
238 tlb_invalidate_one(i);
239 }
240 } else {
241 /*
242 * The virtual address range is large compared to the size of
243 * the TLB. Test every non-wired TLB entry.
244 */
245 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
246 mips_wr_index(i);
247 tlb_read();
248 hi = mips_rd_entryhi();
249 if ((hi & TLBHI_ASID_MASK) == asid && (hi < end_hi ||
250 end == 0)) {
251 /*
252 * If "hi" is a large page that spans
253 * "start_hi", then it must be invalidated.
254 */
255 hi_pagemask = mips_rd_pagemask();
256 if (hi >= (start_hi & ~(hi_pagemask <<
257 TLBMASK_SHIFT)))
258 tlb_invalidate_one(i);
259 }
260 }
261 }
262
263 mips_wr_entryhi(save_asid);
264 intr_restore(s);
265 }
266
267 /* XXX Only if DDB? */
268 void
tlb_save(void)269 tlb_save(void)
270 {
271 unsigned ntlb, i, cpu;
272
273 cpu = PCPU_GET(cpuid);
274 if (num_tlbentries > MIPS_MAX_TLB_ENTRIES)
275 ntlb = MIPS_MAX_TLB_ENTRIES;
276 else
277 ntlb = num_tlbentries;
278 tlb_state[cpu].wired = mips_rd_wired();
279 for (i = 0; i < ntlb; i++) {
280 mips_wr_index(i);
281 tlb_read();
282
283 tlb_state[cpu].entry[i].entryhi = mips_rd_entryhi();
284 tlb_state[cpu].entry[i].pagemask = mips_rd_pagemask();
285 tlb_state[cpu].entry[i].entrylo0 = mips_rd_entrylo0();
286 tlb_state[cpu].entry[i].entrylo1 = mips_rd_entrylo1();
287 }
288 }
289
290 void
tlb_update(struct pmap * pmap,vm_offset_t va,pt_entry_t pte)291 tlb_update(struct pmap *pmap, vm_offset_t va, pt_entry_t pte)
292 {
293 register_t asid;
294 register_t s;
295 int i;
296
297 va &= ~PAGE_MASK;
298 pte &= ~TLBLO_SWBITS_MASK;
299
300 s = intr_disable();
301 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
302
303 mips_wr_pagemask(0);
304 mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
305 tlb_probe();
306 i = mips_rd_index();
307 if (i >= 0) {
308 tlb_read();
309
310 if ((va & PAGE_SIZE) == 0) {
311 mips_wr_entrylo0(pte);
312 } else {
313 mips_wr_entrylo1(pte);
314 }
315 tlb_write_indexed();
316 }
317
318 mips_wr_entryhi(asid);
319 intr_restore(s);
320 }
321
322 static void
tlb_invalidate_one(unsigned i)323 tlb_invalidate_one(unsigned i)
324 {
325 /* XXX an invalid ASID? */
326 mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0));
327 mips_wr_entrylo0(0);
328 mips_wr_entrylo1(0);
329 mips_wr_pagemask(0);
330 mips_wr_index(i);
331 tlb_write_indexed();
332 }
333
334 #ifdef DDB
335 #include <ddb/ddb.h>
336
DB_SHOW_COMMAND(tlb,ddb_dump_tlb)337 DB_SHOW_COMMAND(tlb, ddb_dump_tlb)
338 {
339 register_t ehi, elo0, elo1, epagemask;
340 unsigned i, cpu, ntlb;
341
342 /*
343 * XXX
344 * The worst conversion from hex to decimal ever.
345 */
346 if (have_addr)
347 cpu = ((addr >> 4) % 16) * 10 + (addr % 16);
348 else
349 cpu = PCPU_GET(cpuid);
350
351 if (cpu >= mp_ncpus) {
352 db_printf("Invalid CPU %u\n", cpu);
353 return;
354 }
355 if (num_tlbentries > MIPS_MAX_TLB_ENTRIES) {
356 ntlb = MIPS_MAX_TLB_ENTRIES;
357 db_printf("Warning: Only %d of %d TLB entries saved!\n",
358 ntlb, num_tlbentries);
359 } else
360 ntlb = num_tlbentries;
361
362 if (cpu == PCPU_GET(cpuid))
363 tlb_save();
364
365 db_printf("Beginning TLB dump for CPU %u...\n", cpu);
366 for (i = 0; i < ntlb; i++) {
367 if (i == tlb_state[cpu].wired) {
368 if (i != 0)
369 db_printf("^^^ WIRED ENTRIES ^^^\n");
370 else
371 db_printf("(No wired entries.)\n");
372 }
373
374 /* XXX PageMask. */
375 ehi = tlb_state[cpu].entry[i].entryhi;
376 elo0 = tlb_state[cpu].entry[i].entrylo0;
377 elo1 = tlb_state[cpu].entry[i].entrylo1;
378 epagemask = tlb_state[cpu].entry[i].pagemask;
379
380 if (elo0 == 0 && elo1 == 0)
381 continue;
382
383 db_printf("#%u\t=> %jx (pagemask %jx)\n", i, (intmax_t)ehi, (intmax_t) epagemask);
384 db_printf(" Lo0\t%jx\t(%#jx)\n", (intmax_t)elo0, (intmax_t)TLBLO_PTE_TO_PA(elo0));
385 db_printf(" Lo1\t%jx\t(%#jx)\n", (intmax_t)elo1, (intmax_t)TLBLO_PTE_TO_PA(elo1));
386 }
387 db_printf("Finished.\n");
388 }
389 #endif
390