1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2010 Andreas Tobler
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 #include <sys/param.h>
30 #include <sys/kernel.h>
31 #include <sys/ktr.h>
32 #include <sys/lock.h>
33 #include <sys/rmlock.h>
34 #include <sys/mutex.h>
35 #include <sys/proc.h>
36 #include <sys/sysctl.h>
37 #include <sys/systm.h>
38 #include <sys/vmmeter.h>
39
40 #include <dev/ofw/openfirm.h>
41 #include <machine/ofw_machdep.h>
42
43 #include <vm/vm.h>
44 #include <vm/vm_param.h>
45 #include <vm/vm_kern.h>
46 #include <vm/vm_page.h>
47 #include <vm/vm_map.h>
48 #include <vm/vm_object.h>
49 #include <vm/vm_extern.h>
50 #include <vm/vm_pageout.h>
51 #include <vm/uma.h>
52
53 #include <powerpc/aim/mmu_oea64.h>
54
55 #include "phyp-hvcall.h"
56
57 #define MMU_PHYP_DEBUG 0
58 #define MMU_PHYP_ID "mmu_phyp: "
59 #if MMU_PHYP_DEBUG
60 #define dprintf(fmt, ...) printf(fmt, ## __VA_ARGS__)
61 #define dprintf0(fmt, ...) dprintf(MMU_PHYP_ID fmt, ## __VA_ARGS__)
62 #else
63 #define dprintf(fmt, args...) do { ; } while(0)
64 #define dprintf0(fmt, args...) do { ; } while(0)
65 #endif
66
67 static struct rmlock mphyp_eviction_lock;
68
69 /*
70 * Kernel MMU interface
71 */
72
73 static void mphyp_install(void);
74 static void mphyp_bootstrap(vm_offset_t kernelstart,
75 vm_offset_t kernelend);
76 static void mphyp_cpu_bootstrap(int ap);
77 static void *mphyp_dump_pmap(void *ctx, void *buf,
78 u_long *nbytes);
79 static int64_t mphyp_pte_synch(struct pvo_entry *pvo);
80 static int64_t mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit);
81 static int64_t mphyp_pte_unset(struct pvo_entry *pvo);
82 static int64_t mphyp_pte_insert(struct pvo_entry *pvo);
83 static int64_t mphyp_pte_unset_sp(struct pvo_entry *pvo);
84 static int64_t mphyp_pte_insert_sp(struct pvo_entry *pvo);
85 static int64_t mphyp_pte_replace_sp(struct pvo_entry *pvo);
86
87 static struct pmap_funcs mphyp_methods = {
88 .install = mphyp_install,
89 .bootstrap = mphyp_bootstrap,
90 .cpu_bootstrap = mphyp_cpu_bootstrap,
91 .dumpsys_dump_pmap = mphyp_dump_pmap,
92 };
93
94 static struct moea64_funcs mmu_phyp_funcs = {
95 .pte_synch = mphyp_pte_synch,
96 .pte_clear = mphyp_pte_clear,
97 .pte_unset = mphyp_pte_unset,
98 .pte_insert = mphyp_pte_insert,
99 .pte_unset_sp = mphyp_pte_unset_sp,
100 .pte_insert_sp = mphyp_pte_insert_sp,
101 .pte_replace_sp = mphyp_pte_replace_sp,
102 };
103
104 MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, oea64_mmu);
105
106 static int brokenkvm = 0;
107 static uint64_t final_pteg_count = 0;
108
109 static void
print_kvm_bug_warning(void * data)110 print_kvm_bug_warning(void *data)
111 {
112
113 if (brokenkvm)
114 printf("WARNING: Running on a broken hypervisor that does "
115 "not support mandatory H_CLEAR_MOD and H_CLEAR_REF "
116 "hypercalls. Performance will be suboptimal.\n");
117 }
118
119 SYSINIT(kvmbugwarn1, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 1,
120 print_kvm_bug_warning, NULL);
121 SYSINIT(kvmbugwarn2, SI_SUB_LAST, SI_ORDER_THIRD + 1, print_kvm_bug_warning,
122 NULL);
123
124 static void
mphyp_install(void)125 mphyp_install(void)
126 {
127 char buf[8];
128 uint32_t prop[2];
129 uint32_t nptlp, shift = 0, slb_encoding = 0;
130 uint32_t lp_size, lp_encoding;
131 phandle_t dev, node, root;
132 int idx, len, res;
133 bool has_lp;
134
135 root = OF_peer(0);
136
137 dev = OF_child(root);
138 while (dev != 0) {
139 res = OF_getprop(dev, "name", buf, sizeof(buf));
140 if (res > 0 && strcmp(buf, "cpus") == 0)
141 break;
142 dev = OF_peer(dev);
143 }
144
145 node = OF_child(dev);
146
147 while (node != 0) {
148 res = OF_getprop(node, "device_type", buf, sizeof(buf));
149 if (res > 0 && strcmp(buf, "cpu") == 0)
150 break;
151 node = OF_peer(node);
152 }
153
154 res = OF_getencprop(node, "ibm,pft-size", prop, sizeof(prop));
155 if (res <= 0)
156 panic("mmu_phyp: unknown PFT size");
157 final_pteg_count = 1 << prop[1];
158 res = OF_getencprop(node, "ibm,slb-size", prop, sizeof(prop[0]));
159 if (res > 0)
160 n_slbs = prop[0];
161 dprintf0("slb-size=%i\n", n_slbs);
162
163 /*
164 * Scan the large page size property for PAPR compatible machines.
165 * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties'
166 * for the encoding of the property.
167 */
168
169 len = OF_getproplen(node, "ibm,segment-page-sizes");
170 if (len > 0) {
171 /*
172 * We have to use a variable length array on the stack
173 * since we have very limited stack space.
174 */
175 pcell_t arr[len/sizeof(cell_t)];
176 res = OF_getencprop(node, "ibm,segment-page-sizes", arr,
177 sizeof(arr));
178 len /= 4;
179 idx = 0;
180 has_lp = false;
181 while (len > 0) {
182 shift = arr[idx];
183 slb_encoding = arr[idx + 1];
184 nptlp = arr[idx + 2];
185
186 dprintf0("Segment Page Size: "
187 "%uKB, slb_enc=0x%X: {size, encoding}[%u] =",
188 shift > 10? 1 << (shift-10) : 0,
189 slb_encoding, nptlp);
190
191 idx += 3;
192 len -= 3;
193 while (len > 0 && nptlp) {
194 lp_size = arr[idx];
195 lp_encoding = arr[idx+1];
196
197 dprintf(" {%uKB, 0x%X}",
198 lp_size > 10? 1 << (lp_size-10) : 0,
199 lp_encoding);
200
201 if (slb_encoding == SLBV_L && lp_encoding == 0)
202 has_lp = true;
203
204 if (slb_encoding == SLB_PGSZ_4K_4K &&
205 lp_encoding == LP_4K_16M)
206 moea64_has_lp_4k_16m = true;
207
208 idx += 2;
209 len -= 2;
210 nptlp--;
211 }
212 dprintf("\n");
213 if (has_lp && moea64_has_lp_4k_16m)
214 break;
215 }
216
217 if (has_lp) {
218 moea64_large_page_shift = shift;
219 moea64_large_page_size = 1ULL << lp_size;
220 moea64_large_page_mask = moea64_large_page_size - 1;
221 hw_direct_map = 1;
222 printf(MMU_PHYP_ID
223 "Support for hugepages of %uKB detected\n",
224 moea64_large_page_shift > 10?
225 1 << (moea64_large_page_shift-10) : 0);
226 } else {
227 moea64_large_page_size = 0;
228 moea64_large_page_shift = 0;
229 moea64_large_page_mask = 0;
230 hw_direct_map = 0;
231 printf(MMU_PHYP_ID
232 "Support for hugepages not found\n");
233 }
234 }
235
236 moea64_ops = &mmu_phyp_funcs;
237
238 moea64_install();
239 }
240
241 static void
mphyp_bootstrap(vm_offset_t kernelstart,vm_offset_t kernelend)242 mphyp_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
243 {
244 struct lpte old;
245 uint64_t vsid;
246 int idx;
247
248 rm_init(&mphyp_eviction_lock, "pte eviction");
249
250 moea64_early_bootstrap(kernelstart, kernelend);
251
252 moea64_pteg_count = final_pteg_count / sizeof(struct lpteg);
253
254 /* Clear any old page table entries */
255 for (idx = 0; idx < moea64_pteg_count*8; idx++) {
256 phyp_pft_hcall(H_READ, 0, idx, 0, 0, &old.pte_hi,
257 &old.pte_lo, &old.pte_lo);
258 vsid = (old.pte_hi << (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) >> 28;
259 if (vsid == VSID_VRMA || vsid == 0 /* Older VRMA */)
260 continue;
261
262 if (old.pte_hi & LPTE_VALID)
263 phyp_hcall(H_REMOVE, 0, idx, 0);
264 }
265
266 moea64_mid_bootstrap(kernelstart, kernelend);
267 moea64_late_bootstrap(kernelstart, kernelend);
268
269 /* Test for broken versions of KVM that don't conform to the spec */
270 if (phyp_hcall(H_CLEAR_MOD, 0, 0) == H_FUNCTION)
271 brokenkvm = 1;
272 }
273
274 static void
mphyp_cpu_bootstrap(int ap)275 mphyp_cpu_bootstrap(int ap)
276 {
277 struct slb *slb = PCPU_GET(aim.slb);
278 register_t seg0;
279 int i;
280
281 /*
282 * Install kernel SLB entries
283 */
284
285 __asm __volatile ("slbia");
286 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
287 for (i = 0; i < 64; i++) {
288 if (!(slb[i].slbe & SLBE_VALID))
289 continue;
290
291 __asm __volatile ("slbmte %0, %1" ::
292 "r"(slb[i].slbv), "r"(slb[i].slbe));
293 }
294 }
295
296 static int64_t
mphyp_pte_synch(struct pvo_entry * pvo)297 mphyp_pte_synch(struct pvo_entry *pvo)
298 {
299 struct lpte pte;
300 uint64_t junk;
301
302 __asm __volatile("ptesync");
303 phyp_pft_hcall(H_READ, 0, pvo->pvo_pte.slot, 0, 0, &pte.pte_hi,
304 &pte.pte_lo, &junk);
305 if ((pte.pte_hi & LPTE_AVPN_MASK) !=
306 ((pvo->pvo_vpn >> (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) &
307 LPTE_AVPN_MASK))
308 return (-1);
309 if (!(pte.pte_hi & LPTE_VALID))
310 return (-1);
311
312 return (pte.pte_lo & (LPTE_CHG | LPTE_REF));
313 }
314
315 static int64_t
mphyp_pte_clear(struct pvo_entry * pvo,uint64_t ptebit)316 mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit)
317 {
318 struct rm_priotracker track;
319 int64_t refchg;
320 uint64_t ptelo, junk;
321 int err __diagused;
322
323 /*
324 * This involves two steps (synch and clear) so we need the entry
325 * not to change in the middle. We are protected against deliberate
326 * unset by virtue of holding the pmap lock. Protection against
327 * incidental unset (page table eviction) comes from holding the
328 * shared eviction lock.
329 */
330 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
331 rm_rlock(&mphyp_eviction_lock, &track);
332
333 refchg = mphyp_pte_synch(pvo);
334 if (refchg < 0) {
335 rm_runlock(&mphyp_eviction_lock, &track);
336 return (refchg);
337 }
338
339 if (brokenkvm) {
340 /*
341 * No way to clear either bit, which is total madness.
342 * Pessimistically claim that, once modified, it stays so
343 * forever and that it is never referenced.
344 */
345 rm_runlock(&mphyp_eviction_lock, &track);
346 return (refchg & ~LPTE_REF);
347 }
348
349 if (ptebit & LPTE_CHG) {
350 err = phyp_pft_hcall(H_CLEAR_MOD, 0, pvo->pvo_pte.slot, 0, 0,
351 &ptelo, &junk, &junk);
352 KASSERT(err == H_SUCCESS,
353 ("Error clearing page change bit: %d", err));
354 refchg |= (ptelo & LPTE_CHG);
355 }
356 if (ptebit & LPTE_REF) {
357 err = phyp_pft_hcall(H_CLEAR_REF, 0, pvo->pvo_pte.slot, 0, 0,
358 &ptelo, &junk, &junk);
359 KASSERT(err == H_SUCCESS,
360 ("Error clearing page reference bit: %d", err));
361 refchg |= (ptelo & LPTE_REF);
362 }
363
364 rm_runlock(&mphyp_eviction_lock, &track);
365
366 return (refchg);
367 }
368
369 static int64_t
mphyp_pte_unset(struct pvo_entry * pvo)370 mphyp_pte_unset(struct pvo_entry *pvo)
371 {
372 struct lpte pte;
373 uint64_t junk;
374 int err;
375
376 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
377
378 moea64_pte_from_pvo(pvo, &pte);
379
380 err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot,
381 pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo,
382 &junk);
383 KASSERT(err == H_SUCCESS || err == H_NOT_FOUND,
384 ("Error removing page: %d", err));
385
386 if (err == H_NOT_FOUND) {
387 STAT_MOEA64(moea64_pte_overflow--);
388 return (-1);
389 }
390
391 return (pte.pte_lo & (LPTE_REF | LPTE_CHG));
392 }
393
394 static uintptr_t
mphyp_pte_spillable_ident(uintptr_t ptegbase,struct lpte * to_evict)395 mphyp_pte_spillable_ident(uintptr_t ptegbase, struct lpte *to_evict)
396 {
397 uint64_t slot, junk, k;
398 struct lpte pt;
399 int i, j;
400
401 /* Start at a random slot */
402 i = mftb() % 8;
403 k = -1;
404 for (j = 0; j < 8; j++) {
405 slot = ptegbase + (i + j) % 8;
406 phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi,
407 &pt.pte_lo, &junk);
408
409 if ((pt.pte_hi & (LPTE_WIRED | LPTE_BIG)) != 0)
410 continue;
411
412 /* This is a candidate, so remember it */
413 k = slot;
414
415 /* Try to get a page that has not been used lately */
416 if (!(pt.pte_hi & LPTE_VALID) || !(pt.pte_lo & LPTE_REF)) {
417 memcpy(to_evict, &pt, sizeof(struct lpte));
418 return (k);
419 }
420 }
421
422 if (k == -1)
423 return (k);
424
425 phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi,
426 &to_evict->pte_lo, &junk);
427 return (k);
428 }
429
430 static __inline int64_t
mphyp_pte_insert_locked(struct pvo_entry * pvo,struct lpte * pte)431 mphyp_pte_insert_locked(struct pvo_entry *pvo, struct lpte *pte)
432 {
433 struct lpte evicted;
434 uint64_t index, junk;
435 int64_t result;
436
437 /*
438 * First try primary hash.
439 */
440 pvo->pvo_pte.slot &= ~7UL; /* Base slot address */
441 result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte->pte_hi,
442 pte->pte_lo, &index, &evicted.pte_lo, &junk);
443 if (result == H_SUCCESS) {
444 pvo->pvo_pte.slot = index;
445 return (0);
446 }
447 KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld "
448 "(ptegidx: %#zx/%#lx, PTE %#lx/%#lx", result, pvo->pvo_pte.slot,
449 moea64_pteg_count, pte->pte_hi, pte->pte_lo));
450
451 /*
452 * Next try secondary hash.
453 */
454 pvo->pvo_vaddr ^= PVO_HID;
455 pte->pte_hi ^= LPTE_HID;
456 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
457
458 result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot,
459 pte->pte_hi, pte->pte_lo, &index, &evicted.pte_lo, &junk);
460 if (result == H_SUCCESS) {
461 pvo->pvo_pte.slot = index;
462 return (0);
463 }
464 KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld",
465 result));
466
467 return (-1);
468 }
469
470
471 static __inline int64_t
mphyp_pte_evict_and_insert_locked(struct pvo_entry * pvo,struct lpte * pte)472 mphyp_pte_evict_and_insert_locked(struct pvo_entry *pvo, struct lpte *pte)
473 {
474 struct lpte evicted;
475 uint64_t index, junk, lastptelo;
476 int64_t result;
477
478 evicted.pte_hi = 0;
479
480 index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted);
481 if (index == -1L) {
482 /* Try other hash table? */
483 pvo->pvo_vaddr ^= PVO_HID;
484 pte->pte_hi ^= LPTE_HID;
485 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
486 index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted);
487 }
488
489 if (index == -1L) {
490 /* No freeable slots in either PTEG? We're hosed. */
491 rm_wunlock(&mphyp_eviction_lock);
492 panic("mphyp_pte_insert: overflow");
493 return (-1);
494 }
495
496 /* Victim acquired: update page before waving goodbye */
497 if (evicted.pte_hi & LPTE_VALID) {
498 result = phyp_pft_hcall(H_REMOVE, H_AVPN, index,
499 evicted.pte_hi & LPTE_AVPN_MASK, 0, &junk, &lastptelo,
500 &junk);
501 STAT_MOEA64(moea64_pte_overflow++);
502 KASSERT(result == H_SUCCESS || result == H_NOT_FOUND,
503 ("Error evicting page: %d", (int)result));
504 }
505
506 /*
507 * Set the new PTE.
508 */
509 result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte->pte_hi,
510 pte->pte_lo, &index, &evicted.pte_lo, &junk);
511
512 pvo->pvo_pte.slot = index;
513 if (result == H_SUCCESS)
514 return (0);
515
516 rm_wunlock(&mphyp_eviction_lock);
517 panic("Page replacement error: %ld", result);
518 return (result);
519 }
520
521 static int64_t
mphyp_pte_insert(struct pvo_entry * pvo)522 mphyp_pte_insert(struct pvo_entry *pvo)
523 {
524 struct rm_priotracker track;
525 int64_t ret;
526 struct lpte pte;
527
528 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
529
530 /* Initialize PTE */
531 moea64_pte_from_pvo(pvo, &pte);
532
533 /* Make sure further insertion is locked out during evictions */
534 rm_rlock(&mphyp_eviction_lock, &track);
535
536 ret = mphyp_pte_insert_locked(pvo, &pte);
537 rm_runlock(&mphyp_eviction_lock, &track);
538
539 if (ret == -1) {
540 /*
541 * Out of luck. Find a PTE to sacrifice.
542 */
543
544 /* Lock out all insertions for a bit */
545 rm_wlock(&mphyp_eviction_lock);
546 ret = mphyp_pte_evict_and_insert_locked(pvo, &pte);
547 rm_wunlock(&mphyp_eviction_lock); /* All clear */
548 }
549
550 return (ret);
551 }
552
553 static void *
mphyp_dump_pmap(void * ctx,void * buf,u_long * nbytes)554 mphyp_dump_pmap(void *ctx, void *buf, u_long *nbytes)
555 {
556 struct dump_context *dctx;
557 struct lpte p, *pbuf;
558 int bufidx;
559 uint64_t junk;
560 u_long ptex, ptex_end;
561
562 dctx = (struct dump_context *)ctx;
563 pbuf = (struct lpte *)buf;
564 bufidx = 0;
565 ptex = dctx->ptex;
566 ptex_end = ptex + dctx->blksz / sizeof(struct lpte);
567 ptex_end = MIN(ptex_end, dctx->ptex_end);
568 *nbytes = (ptex_end - ptex) * sizeof(struct lpte);
569
570 if (*nbytes == 0)
571 return (NULL);
572
573 for (; ptex < ptex_end; ptex++) {
574 phyp_pft_hcall(H_READ, 0, ptex, 0, 0,
575 &p.pte_hi, &p.pte_lo, &junk);
576 pbuf[bufidx++] = p;
577 }
578
579 dctx->ptex = ptex;
580 return (buf);
581 }
582
583 static int64_t
mphyp_pte_unset_sp(struct pvo_entry * pvo)584 mphyp_pte_unset_sp(struct pvo_entry *pvo)
585 {
586 struct lpte pte;
587 uint64_t junk, refchg;
588 int err;
589 vm_offset_t eva;
590 pmap_t pm __diagused;
591
592 pm = pvo->pvo_pmap;
593 PMAP_LOCK_ASSERT(pm, MA_OWNED);
594 KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0,
595 ("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));
596
597 refchg = 0;
598 eva = PVO_VADDR(pvo) + HPT_SP_SIZE;
599
600 for (; pvo != NULL && PVO_VADDR(pvo) < eva;
601 pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
602 moea64_pte_from_pvo(pvo, &pte);
603
604 err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot,
605 pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo,
606 &junk);
607 KASSERT(err == H_SUCCESS || err == H_NOT_FOUND,
608 ("Error removing page: %d", err));
609
610 if (err == H_NOT_FOUND)
611 STAT_MOEA64(moea64_pte_overflow--);
612 refchg |= pte.pte_lo & (LPTE_REF | LPTE_CHG);
613 }
614
615 return (refchg);
616 }
617
618 static int64_t
mphyp_pte_insert_sp(struct pvo_entry * pvo)619 mphyp_pte_insert_sp(struct pvo_entry *pvo)
620 {
621 struct rm_priotracker track;
622 int64_t ret;
623 struct lpte pte;
624 vm_offset_t eva;
625 pmap_t pm __diagused;
626
627 pm = pvo->pvo_pmap;
628 PMAP_LOCK_ASSERT(pm, MA_OWNED);
629 KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0,
630 ("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));
631
632 eva = PVO_VADDR(pvo) + HPT_SP_SIZE;
633
634 /* Make sure further insertion is locked out during evictions */
635 rm_rlock(&mphyp_eviction_lock, &track);
636
637 for (; pvo != NULL && PVO_VADDR(pvo) < eva;
638 pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
639 /* Initialize PTE */
640 moea64_pte_from_pvo(pvo, &pte);
641
642 ret = mphyp_pte_insert_locked(pvo, &pte);
643 if (ret == -1) {
644 /*
645 * Out of luck. Find a PTE to sacrifice.
646 */
647
648 /* Lock out all insertions for a bit */
649 rm_runlock(&mphyp_eviction_lock, &track);
650 rm_wlock(&mphyp_eviction_lock);
651 mphyp_pte_evict_and_insert_locked(pvo, &pte);
652 rm_wunlock(&mphyp_eviction_lock); /* All clear */
653 rm_rlock(&mphyp_eviction_lock, &track);
654 }
655 }
656
657 rm_runlock(&mphyp_eviction_lock, &track);
658 return (0);
659 }
660
661 static int64_t
mphyp_pte_replace_sp(struct pvo_entry * pvo)662 mphyp_pte_replace_sp(struct pvo_entry *pvo)
663 {
664 int64_t refchg;
665
666 refchg = mphyp_pte_unset_sp(pvo);
667 mphyp_pte_insert_sp(pvo);
668 return (refchg);
669 }
670