1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/pagewalk.h> 3 #include <linux/mm_inline.h> 4 #include <linux/hugetlb.h> 5 #include <linux/huge_mm.h> 6 #include <linux/mount.h> 7 #include <linux/ksm.h> 8 #include <linux/seq_file.h> 9 #include <linux/highmem.h> 10 #include <linux/ptrace.h> 11 #include <linux/slab.h> 12 #include <linux/pagemap.h> 13 #include <linux/mempolicy.h> 14 #include <linux/rmap.h> 15 #include <linux/swap.h> 16 #include <linux/sched/mm.h> 17 #include <linux/swapops.h> 18 #include <linux/mmu_notifier.h> 19 #include <linux/page_idle.h> 20 #include <linux/shmem_fs.h> 21 #include <linux/uaccess.h> 22 #include <linux/pkeys.h> 23 #include <linux/minmax.h> 24 #include <linux/overflow.h> 25 26 #include <asm/elf.h> 27 #include <asm/tlb.h> 28 #include <asm/tlbflush.h> 29 #include "internal.h" 30 31 #define SEQ_PUT_DEC(str, val) \ 32 seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) 33 void task_mem(struct seq_file *m, struct mm_struct *mm) 34 { 35 unsigned long text, lib, swap, anon, file, shmem; 36 unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss; 37 38 anon = get_mm_counter(mm, MM_ANONPAGES); 39 file = get_mm_counter(mm, MM_FILEPAGES); 40 shmem = get_mm_counter(mm, MM_SHMEMPAGES); 41 42 /* 43 * Note: to minimize their overhead, mm maintains hiwater_vm and 44 * hiwater_rss only when about to *lower* total_vm or rss. Any 45 * collector of these hiwater stats must therefore get total_vm 46 * and rss too, which will usually be the higher. Barriers? not 47 * worth the effort, such snapshots can always be inconsistent. 48 */ 49 hiwater_vm = total_vm = mm->total_vm; 50 if (hiwater_vm < mm->hiwater_vm) 51 hiwater_vm = mm->hiwater_vm; 52 hiwater_rss = total_rss = anon + file + shmem; 53 if (hiwater_rss < mm->hiwater_rss) 54 hiwater_rss = mm->hiwater_rss; 55 56 /* split executable areas between text and lib */ 57 text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); 58 text = min(text, mm->exec_vm << PAGE_SHIFT); 59 lib = (mm->exec_vm << PAGE_SHIFT) - text; 60 61 swap = get_mm_counter(mm, MM_SWAPENTS); 62 SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); 63 SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); 64 SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); 65 SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); 66 SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); 67 SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); 68 SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); 69 SEQ_PUT_DEC(" kB\nRssFile:\t", file); 70 SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem); 71 SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm); 72 SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm); 73 seq_put_decimal_ull_width(m, 74 " kB\nVmExe:\t", text >> 10, 8); 75 seq_put_decimal_ull_width(m, 76 " kB\nVmLib:\t", lib >> 10, 8); 77 seq_put_decimal_ull_width(m, 78 " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8); 79 SEQ_PUT_DEC(" kB\nVmSwap:\t", swap); 80 seq_puts(m, " kB\n"); 81 hugetlb_report_usage(m, mm); 82 } 83 #undef SEQ_PUT_DEC 84 85 unsigned long task_vsize(struct mm_struct *mm) 86 { 87 return PAGE_SIZE * mm->total_vm; 88 } 89 90 unsigned long task_statm(struct mm_struct *mm, 91 unsigned long *shared, unsigned long *text, 92 unsigned long *data, unsigned long *resident) 93 { 94 *shared = get_mm_counter(mm, MM_FILEPAGES) + 95 get_mm_counter(mm, MM_SHMEMPAGES); 96 *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) 97 >> PAGE_SHIFT; 98 *data = mm->data_vm + mm->stack_vm; 99 *resident = *shared + get_mm_counter(mm, MM_ANONPAGES); 100 return mm->total_vm; 101 } 102 103 #ifdef CONFIG_NUMA 104 /* 105 * Save get_task_policy() for show_numa_map(). 106 */ 107 static void hold_task_mempolicy(struct proc_maps_private *priv) 108 { 109 struct task_struct *task = priv->task; 110 111 task_lock(task); 112 priv->task_mempolicy = get_task_policy(task); 113 mpol_get(priv->task_mempolicy); 114 task_unlock(task); 115 } 116 static void release_task_mempolicy(struct proc_maps_private *priv) 117 { 118 mpol_put(priv->task_mempolicy); 119 } 120 #else 121 static void hold_task_mempolicy(struct proc_maps_private *priv) 122 { 123 } 124 static void release_task_mempolicy(struct proc_maps_private *priv) 125 { 126 } 127 #endif 128 129 static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv, 130 loff_t *ppos) 131 { 132 struct vm_area_struct *vma = vma_next(&priv->iter); 133 134 if (vma) { 135 *ppos = vma->vm_start; 136 } else { 137 *ppos = -2UL; 138 vma = get_gate_vma(priv->mm); 139 } 140 141 return vma; 142 } 143 144 static void *m_start(struct seq_file *m, loff_t *ppos) 145 { 146 struct proc_maps_private *priv = m->private; 147 unsigned long last_addr = *ppos; 148 struct mm_struct *mm; 149 150 /* See m_next(). Zero at the start or after lseek. */ 151 if (last_addr == -1UL) 152 return NULL; 153 154 priv->task = get_proc_task(priv->inode); 155 if (!priv->task) 156 return ERR_PTR(-ESRCH); 157 158 mm = priv->mm; 159 if (!mm || !mmget_not_zero(mm)) { 160 put_task_struct(priv->task); 161 priv->task = NULL; 162 return NULL; 163 } 164 165 if (mmap_read_lock_killable(mm)) { 166 mmput(mm); 167 put_task_struct(priv->task); 168 priv->task = NULL; 169 return ERR_PTR(-EINTR); 170 } 171 172 vma_iter_init(&priv->iter, mm, last_addr); 173 hold_task_mempolicy(priv); 174 if (last_addr == -2UL) 175 return get_gate_vma(mm); 176 177 return proc_get_vma(priv, ppos); 178 } 179 180 static void *m_next(struct seq_file *m, void *v, loff_t *ppos) 181 { 182 if (*ppos == -2UL) { 183 *ppos = -1UL; 184 return NULL; 185 } 186 return proc_get_vma(m->private, ppos); 187 } 188 189 static void m_stop(struct seq_file *m, void *v) 190 { 191 struct proc_maps_private *priv = m->private; 192 struct mm_struct *mm = priv->mm; 193 194 if (!priv->task) 195 return; 196 197 release_task_mempolicy(priv); 198 mmap_read_unlock(mm); 199 mmput(mm); 200 put_task_struct(priv->task); 201 priv->task = NULL; 202 } 203 204 static int proc_maps_open(struct inode *inode, struct file *file, 205 const struct seq_operations *ops, int psize) 206 { 207 struct proc_maps_private *priv = __seq_open_private(file, ops, psize); 208 209 if (!priv) 210 return -ENOMEM; 211 212 priv->inode = inode; 213 priv->mm = proc_mem_open(inode, PTRACE_MODE_READ); 214 if (IS_ERR(priv->mm)) { 215 int err = PTR_ERR(priv->mm); 216 217 seq_release_private(inode, file); 218 return err; 219 } 220 221 return 0; 222 } 223 224 static int proc_map_release(struct inode *inode, struct file *file) 225 { 226 struct seq_file *seq = file->private_data; 227 struct proc_maps_private *priv = seq->private; 228 229 if (priv->mm) 230 mmdrop(priv->mm); 231 232 return seq_release_private(inode, file); 233 } 234 235 static int do_maps_open(struct inode *inode, struct file *file, 236 const struct seq_operations *ops) 237 { 238 return proc_maps_open(inode, file, ops, 239 sizeof(struct proc_maps_private)); 240 } 241 242 static void get_vma_name(struct vm_area_struct *vma, 243 const struct path **path, 244 const char **name, 245 const char **name_fmt) 246 { 247 struct anon_vma_name *anon_name = vma->vm_mm ? anon_vma_name(vma) : NULL; 248 249 *name = NULL; 250 *path = NULL; 251 *name_fmt = NULL; 252 253 /* 254 * Print the dentry name for named mappings, and a 255 * special [heap] marker for the heap: 256 */ 257 if (vma->vm_file) { 258 /* 259 * If user named this anon shared memory via 260 * prctl(PR_SET_VMA ..., use the provided name. 261 */ 262 if (anon_name) { 263 *name_fmt = "[anon_shmem:%s]"; 264 *name = anon_name->name; 265 } else { 266 *path = file_user_path(vma->vm_file); 267 } 268 return; 269 } 270 271 if (vma->vm_ops && vma->vm_ops->name) { 272 *name = vma->vm_ops->name(vma); 273 if (*name) 274 return; 275 } 276 277 *name = arch_vma_name(vma); 278 if (*name) 279 return; 280 281 if (!vma->vm_mm) { 282 *name = "[vdso]"; 283 return; 284 } 285 286 if (vma_is_initial_heap(vma)) { 287 *name = "[heap]"; 288 return; 289 } 290 291 if (vma_is_initial_stack(vma)) { 292 *name = "[stack]"; 293 return; 294 } 295 296 if (anon_name) { 297 *name_fmt = "[anon:%s]"; 298 *name = anon_name->name; 299 return; 300 } 301 } 302 303 static void show_vma_header_prefix(struct seq_file *m, 304 unsigned long start, unsigned long end, 305 vm_flags_t flags, unsigned long long pgoff, 306 dev_t dev, unsigned long ino) 307 { 308 seq_setwidth(m, 25 + sizeof(void *) * 6 - 1); 309 seq_put_hex_ll(m, NULL, start, 8); 310 seq_put_hex_ll(m, "-", end, 8); 311 seq_putc(m, ' '); 312 seq_putc(m, flags & VM_READ ? 'r' : '-'); 313 seq_putc(m, flags & VM_WRITE ? 'w' : '-'); 314 seq_putc(m, flags & VM_EXEC ? 'x' : '-'); 315 seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p'); 316 seq_put_hex_ll(m, " ", pgoff, 8); 317 seq_put_hex_ll(m, " ", MAJOR(dev), 2); 318 seq_put_hex_ll(m, ":", MINOR(dev), 2); 319 seq_put_decimal_ull(m, " ", ino); 320 seq_putc(m, ' '); 321 } 322 323 static void 324 show_map_vma(struct seq_file *m, struct vm_area_struct *vma) 325 { 326 const struct path *path; 327 const char *name_fmt, *name; 328 vm_flags_t flags = vma->vm_flags; 329 unsigned long ino = 0; 330 unsigned long long pgoff = 0; 331 unsigned long start, end; 332 dev_t dev = 0; 333 334 if (vma->vm_file) { 335 const struct inode *inode = file_user_inode(vma->vm_file); 336 337 dev = inode->i_sb->s_dev; 338 ino = inode->i_ino; 339 pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; 340 } 341 342 start = vma->vm_start; 343 end = vma->vm_end; 344 show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino); 345 346 get_vma_name(vma, &path, &name, &name_fmt); 347 if (path) { 348 seq_pad(m, ' '); 349 seq_path(m, path, "\n"); 350 } else if (name_fmt) { 351 seq_pad(m, ' '); 352 seq_printf(m, name_fmt, name); 353 } else if (name) { 354 seq_pad(m, ' '); 355 seq_puts(m, name); 356 } 357 seq_putc(m, '\n'); 358 } 359 360 static int show_map(struct seq_file *m, void *v) 361 { 362 show_map_vma(m, v); 363 return 0; 364 } 365 366 static const struct seq_operations proc_pid_maps_op = { 367 .start = m_start, 368 .next = m_next, 369 .stop = m_stop, 370 .show = show_map 371 }; 372 373 static int pid_maps_open(struct inode *inode, struct file *file) 374 { 375 return do_maps_open(inode, file, &proc_pid_maps_op); 376 } 377 378 #define PROCMAP_QUERY_VMA_FLAGS ( \ 379 PROCMAP_QUERY_VMA_READABLE | \ 380 PROCMAP_QUERY_VMA_WRITABLE | \ 381 PROCMAP_QUERY_VMA_EXECUTABLE | \ 382 PROCMAP_QUERY_VMA_SHARED \ 383 ) 384 385 #define PROCMAP_QUERY_VALID_FLAGS_MASK ( \ 386 PROCMAP_QUERY_COVERING_OR_NEXT_VMA | \ 387 PROCMAP_QUERY_FILE_BACKED_VMA | \ 388 PROCMAP_QUERY_VMA_FLAGS \ 389 ) 390 391 static int query_vma_setup(struct mm_struct *mm) 392 { 393 return mmap_read_lock_killable(mm); 394 } 395 396 static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct *vma) 397 { 398 mmap_read_unlock(mm); 399 } 400 401 static struct vm_area_struct *query_vma_find_by_addr(struct mm_struct *mm, unsigned long addr) 402 { 403 return find_vma(mm, addr); 404 } 405 406 static struct vm_area_struct *query_matching_vma(struct mm_struct *mm, 407 unsigned long addr, u32 flags) 408 { 409 struct vm_area_struct *vma; 410 411 next_vma: 412 vma = query_vma_find_by_addr(mm, addr); 413 if (!vma) 414 goto no_vma; 415 416 /* user requested only file-backed VMA, keep iterating */ 417 if ((flags & PROCMAP_QUERY_FILE_BACKED_VMA) && !vma->vm_file) 418 goto skip_vma; 419 420 /* VMA permissions should satisfy query flags */ 421 if (flags & PROCMAP_QUERY_VMA_FLAGS) { 422 u32 perm = 0; 423 424 if (flags & PROCMAP_QUERY_VMA_READABLE) 425 perm |= VM_READ; 426 if (flags & PROCMAP_QUERY_VMA_WRITABLE) 427 perm |= VM_WRITE; 428 if (flags & PROCMAP_QUERY_VMA_EXECUTABLE) 429 perm |= VM_EXEC; 430 if (flags & PROCMAP_QUERY_VMA_SHARED) 431 perm |= VM_MAYSHARE; 432 433 if ((vma->vm_flags & perm) != perm) 434 goto skip_vma; 435 } 436 437 /* found covering VMA or user is OK with the matching next VMA */ 438 if ((flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) || vma->vm_start <= addr) 439 return vma; 440 441 skip_vma: 442 /* 443 * If the user needs closest matching VMA, keep iterating. 444 */ 445 addr = vma->vm_end; 446 if (flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) 447 goto next_vma; 448 no_vma: 449 return ERR_PTR(-ENOENT); 450 } 451 452 static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg) 453 { 454 struct procmap_query karg; 455 struct vm_area_struct *vma; 456 struct mm_struct *mm; 457 const char *name = NULL; 458 char *name_buf = NULL; 459 __u64 usize; 460 int err; 461 462 if (copy_from_user(&usize, (void __user *)uarg, sizeof(usize))) 463 return -EFAULT; 464 /* argument struct can never be that large, reject abuse */ 465 if (usize > PAGE_SIZE) 466 return -E2BIG; 467 /* argument struct should have at least query_flags and query_addr fields */ 468 if (usize < offsetofend(struct procmap_query, query_addr)) 469 return -EINVAL; 470 err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize); 471 if (err) 472 return err; 473 474 /* reject unknown flags */ 475 if (karg.query_flags & ~PROCMAP_QUERY_VALID_FLAGS_MASK) 476 return -EINVAL; 477 /* either both buffer address and size are set, or both should be zero */ 478 if (!!karg.vma_name_size != !!karg.vma_name_addr) 479 return -EINVAL; 480 481 mm = priv->mm; 482 if (!mm || !mmget_not_zero(mm)) 483 return -ESRCH; 484 485 err = query_vma_setup(mm); 486 if (err) { 487 mmput(mm); 488 return err; 489 } 490 491 vma = query_matching_vma(mm, karg.query_addr, karg.query_flags); 492 if (IS_ERR(vma)) { 493 err = PTR_ERR(vma); 494 vma = NULL; 495 goto out; 496 } 497 498 karg.vma_start = vma->vm_start; 499 karg.vma_end = vma->vm_end; 500 501 karg.vma_flags = 0; 502 if (vma->vm_flags & VM_READ) 503 karg.vma_flags |= PROCMAP_QUERY_VMA_READABLE; 504 if (vma->vm_flags & VM_WRITE) 505 karg.vma_flags |= PROCMAP_QUERY_VMA_WRITABLE; 506 if (vma->vm_flags & VM_EXEC) 507 karg.vma_flags |= PROCMAP_QUERY_VMA_EXECUTABLE; 508 if (vma->vm_flags & VM_MAYSHARE) 509 karg.vma_flags |= PROCMAP_QUERY_VMA_SHARED; 510 511 karg.vma_page_size = vma_kernel_pagesize(vma); 512 513 if (vma->vm_file) { 514 const struct inode *inode = file_user_inode(vma->vm_file); 515 516 karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; 517 karg.dev_major = MAJOR(inode->i_sb->s_dev); 518 karg.dev_minor = MINOR(inode->i_sb->s_dev); 519 karg.inode = inode->i_ino; 520 } else { 521 karg.vma_offset = 0; 522 karg.dev_major = 0; 523 karg.dev_minor = 0; 524 karg.inode = 0; 525 } 526 527 if (karg.build_id_size) { 528 __u32 build_id_sz; 529 530 err = build_id_parse(vma, build_id_buf, &build_id_sz); 531 if (err) { 532 karg.build_id_size = 0; 533 } else { 534 if (karg.build_id_size < build_id_sz) { 535 err = -ENAMETOOLONG; 536 goto out; 537 } 538 karg.build_id_size = build_id_sz; 539 } 540 } 541 542 if (karg.vma_name_size) { 543 size_t name_buf_sz = min_t(size_t, PATH_MAX, karg.vma_name_size); 544 const struct path *path; 545 const char *name_fmt; 546 size_t name_sz = 0; 547 548 get_vma_name(vma, &path, &name, &name_fmt); 549 550 if (path || name_fmt || name) { 551 name_buf = kmalloc(name_buf_sz, GFP_KERNEL); 552 if (!name_buf) { 553 err = -ENOMEM; 554 goto out; 555 } 556 } 557 if (path) { 558 name = d_path(path, name_buf, name_buf_sz); 559 if (IS_ERR(name)) { 560 err = PTR_ERR(name); 561 goto out; 562 } 563 name_sz = name_buf + name_buf_sz - name; 564 } else if (name || name_fmt) { 565 name_sz = 1 + snprintf(name_buf, name_buf_sz, name_fmt ?: "%s", name); 566 name = name_buf; 567 } 568 if (name_sz > name_buf_sz) { 569 err = -ENAMETOOLONG; 570 goto out; 571 } 572 karg.vma_name_size = name_sz; 573 } 574 575 /* unlock vma or mmap_lock, and put mm_struct before copying data to user */ 576 query_vma_teardown(mm, vma); 577 mmput(mm); 578 579 if (karg.vma_name_size && copy_to_user((void __user *)karg.vma_name_addr, 580 name, karg.vma_name_size)) { 581 kfree(name_buf); 582 return -EFAULT; 583 } 584 kfree(name_buf); 585 586 if (copy_to_user(uarg, &karg, min_t(size_t, sizeof(karg), usize))) 587 return -EFAULT; 588 589 return 0; 590 591 out: 592 query_vma_teardown(mm, vma); 593 mmput(mm); 594 kfree(name_buf); 595 return err; 596 } 597 598 static long procfs_procmap_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 599 { 600 struct seq_file *seq = file->private_data; 601 struct proc_maps_private *priv = seq->private; 602 603 switch (cmd) { 604 case PROCMAP_QUERY: 605 return do_procmap_query(priv, (void __user *)arg); 606 default: 607 return -ENOIOCTLCMD; 608 } 609 } 610 611 const struct file_operations proc_pid_maps_operations = { 612 .open = pid_maps_open, 613 .read = seq_read, 614 .llseek = seq_lseek, 615 .release = proc_map_release, 616 .unlocked_ioctl = procfs_procmap_ioctl, 617 .compat_ioctl = compat_ptr_ioctl, 618 }; 619 620 /* 621 * Proportional Set Size(PSS): my share of RSS. 622 * 623 * PSS of a process is the count of pages it has in memory, where each 624 * page is divided by the number of processes sharing it. So if a 625 * process has 1000 pages all to itself, and 1000 shared with one other 626 * process, its PSS will be 1500. 627 * 628 * To keep (accumulated) division errors low, we adopt a 64bit 629 * fixed-point pss counter to minimize division errors. So (pss >> 630 * PSS_SHIFT) would be the real byte count. 631 * 632 * A shift of 12 before division means (assuming 4K page size): 633 * - 1M 3-user-pages add up to 8KB errors; 634 * - supports mapcount up to 2^24, or 16M; 635 * - supports PSS up to 2^52 bytes, or 4PB. 636 */ 637 #define PSS_SHIFT 12 638 639 #ifdef CONFIG_PROC_PAGE_MONITOR 640 struct mem_size_stats { 641 unsigned long resident; 642 unsigned long shared_clean; 643 unsigned long shared_dirty; 644 unsigned long private_clean; 645 unsigned long private_dirty; 646 unsigned long referenced; 647 unsigned long anonymous; 648 unsigned long lazyfree; 649 unsigned long anonymous_thp; 650 unsigned long shmem_thp; 651 unsigned long file_thp; 652 unsigned long swap; 653 unsigned long shared_hugetlb; 654 unsigned long private_hugetlb; 655 unsigned long ksm; 656 u64 pss; 657 u64 pss_anon; 658 u64 pss_file; 659 u64 pss_shmem; 660 u64 pss_dirty; 661 u64 pss_locked; 662 u64 swap_pss; 663 }; 664 665 static void smaps_page_accumulate(struct mem_size_stats *mss, 666 struct folio *folio, unsigned long size, unsigned long pss, 667 bool dirty, bool locked, bool private) 668 { 669 mss->pss += pss; 670 671 if (folio_test_anon(folio)) 672 mss->pss_anon += pss; 673 else if (folio_test_swapbacked(folio)) 674 mss->pss_shmem += pss; 675 else 676 mss->pss_file += pss; 677 678 if (locked) 679 mss->pss_locked += pss; 680 681 if (dirty || folio_test_dirty(folio)) { 682 mss->pss_dirty += pss; 683 if (private) 684 mss->private_dirty += size; 685 else 686 mss->shared_dirty += size; 687 } else { 688 if (private) 689 mss->private_clean += size; 690 else 691 mss->shared_clean += size; 692 } 693 } 694 695 static void smaps_account(struct mem_size_stats *mss, struct page *page, 696 bool compound, bool young, bool dirty, bool locked, 697 bool present) 698 { 699 struct folio *folio = page_folio(page); 700 int i, nr = compound ? compound_nr(page) : 1; 701 unsigned long size = nr * PAGE_SIZE; 702 703 /* 704 * First accumulate quantities that depend only on |size| and the type 705 * of the compound page. 706 */ 707 if (folio_test_anon(folio)) { 708 mss->anonymous += size; 709 if (!folio_test_swapbacked(folio) && !dirty && 710 !folio_test_dirty(folio)) 711 mss->lazyfree += size; 712 } 713 714 if (folio_test_ksm(folio)) 715 mss->ksm += size; 716 717 mss->resident += size; 718 /* Accumulate the size in pages that have been accessed. */ 719 if (young || folio_test_young(folio) || folio_test_referenced(folio)) 720 mss->referenced += size; 721 722 /* 723 * Then accumulate quantities that may depend on sharing, or that may 724 * differ page-by-page. 725 * 726 * refcount == 1 for present entries guarantees that the folio is mapped 727 * exactly once. For large folios this implies that exactly one 728 * PTE/PMD/... maps (a part of) this folio. 729 * 730 * Treat all non-present entries (where relying on the mapcount and 731 * refcount doesn't make sense) as "maybe shared, but not sure how 732 * often". We treat device private entries as being fake-present. 733 * 734 * Note that it would not be safe to read the mapcount especially for 735 * pages referenced by migration entries, even with the PTL held. 736 */ 737 if (folio_ref_count(folio) == 1 || !present) { 738 smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT, 739 dirty, locked, present); 740 return; 741 } 742 /* 743 * We obtain a snapshot of the mapcount. Without holding the folio lock 744 * this snapshot can be slightly wrong as we cannot always read the 745 * mapcount atomically. 746 */ 747 for (i = 0; i < nr; i++, page++) { 748 int mapcount = folio_precise_page_mapcount(folio, page); 749 unsigned long pss = PAGE_SIZE << PSS_SHIFT; 750 if (mapcount >= 2) 751 pss /= mapcount; 752 smaps_page_accumulate(mss, folio, PAGE_SIZE, pss, 753 dirty, locked, mapcount < 2); 754 } 755 } 756 757 #ifdef CONFIG_SHMEM 758 static int smaps_pte_hole(unsigned long addr, unsigned long end, 759 __always_unused int depth, struct mm_walk *walk) 760 { 761 struct mem_size_stats *mss = walk->private; 762 struct vm_area_struct *vma = walk->vma; 763 764 mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, 765 linear_page_index(vma, addr), 766 linear_page_index(vma, end)); 767 768 return 0; 769 } 770 #else 771 #define smaps_pte_hole NULL 772 #endif /* CONFIG_SHMEM */ 773 774 static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) 775 { 776 #ifdef CONFIG_SHMEM 777 if (walk->ops->pte_hole) { 778 /* depth is not used */ 779 smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); 780 } 781 #endif 782 } 783 784 static void smaps_pte_entry(pte_t *pte, unsigned long addr, 785 struct mm_walk *walk) 786 { 787 struct mem_size_stats *mss = walk->private; 788 struct vm_area_struct *vma = walk->vma; 789 bool locked = !!(vma->vm_flags & VM_LOCKED); 790 struct page *page = NULL; 791 bool present = false, young = false, dirty = false; 792 pte_t ptent = ptep_get(pte); 793 794 if (pte_present(ptent)) { 795 page = vm_normal_page(vma, addr, ptent); 796 young = pte_young(ptent); 797 dirty = pte_dirty(ptent); 798 present = true; 799 } else if (is_swap_pte(ptent)) { 800 swp_entry_t swpent = pte_to_swp_entry(ptent); 801 802 if (!non_swap_entry(swpent)) { 803 int mapcount; 804 805 mss->swap += PAGE_SIZE; 806 mapcount = swp_swapcount(swpent); 807 if (mapcount >= 2) { 808 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; 809 810 do_div(pss_delta, mapcount); 811 mss->swap_pss += pss_delta; 812 } else { 813 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; 814 } 815 } else if (is_pfn_swap_entry(swpent)) { 816 if (is_device_private_entry(swpent)) 817 present = true; 818 page = pfn_swap_entry_to_page(swpent); 819 } 820 } else { 821 smaps_pte_hole_lookup(addr, walk); 822 return; 823 } 824 825 if (!page) 826 return; 827 828 smaps_account(mss, page, false, young, dirty, locked, present); 829 } 830 831 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 832 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, 833 struct mm_walk *walk) 834 { 835 struct mem_size_stats *mss = walk->private; 836 struct vm_area_struct *vma = walk->vma; 837 bool locked = !!(vma->vm_flags & VM_LOCKED); 838 struct page *page = NULL; 839 bool present = false; 840 struct folio *folio; 841 842 if (pmd_present(*pmd)) { 843 page = vm_normal_page_pmd(vma, addr, *pmd); 844 present = true; 845 } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) { 846 swp_entry_t entry = pmd_to_swp_entry(*pmd); 847 848 if (is_pfn_swap_entry(entry)) 849 page = pfn_swap_entry_to_page(entry); 850 } 851 if (IS_ERR_OR_NULL(page)) 852 return; 853 folio = page_folio(page); 854 if (folio_test_anon(folio)) 855 mss->anonymous_thp += HPAGE_PMD_SIZE; 856 else if (folio_test_swapbacked(folio)) 857 mss->shmem_thp += HPAGE_PMD_SIZE; 858 else if (folio_is_zone_device(folio)) 859 /* pass */; 860 else 861 mss->file_thp += HPAGE_PMD_SIZE; 862 863 smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), 864 locked, present); 865 } 866 #else 867 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, 868 struct mm_walk *walk) 869 { 870 } 871 #endif 872 873 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 874 struct mm_walk *walk) 875 { 876 struct vm_area_struct *vma = walk->vma; 877 pte_t *pte; 878 spinlock_t *ptl; 879 880 ptl = pmd_trans_huge_lock(pmd, vma); 881 if (ptl) { 882 smaps_pmd_entry(pmd, addr, walk); 883 spin_unlock(ptl); 884 goto out; 885 } 886 887 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 888 if (!pte) { 889 walk->action = ACTION_AGAIN; 890 return 0; 891 } 892 for (; addr != end; pte++, addr += PAGE_SIZE) 893 smaps_pte_entry(pte, addr, walk); 894 pte_unmap_unlock(pte - 1, ptl); 895 out: 896 cond_resched(); 897 return 0; 898 } 899 900 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) 901 { 902 /* 903 * Don't forget to update Documentation/ on changes. 904 */ 905 static const char mnemonics[BITS_PER_LONG][2] = { 906 /* 907 * In case if we meet a flag we don't know about. 908 */ 909 [0 ... (BITS_PER_LONG-1)] = "??", 910 911 [ilog2(VM_READ)] = "rd", 912 [ilog2(VM_WRITE)] = "wr", 913 [ilog2(VM_EXEC)] = "ex", 914 [ilog2(VM_SHARED)] = "sh", 915 [ilog2(VM_MAYREAD)] = "mr", 916 [ilog2(VM_MAYWRITE)] = "mw", 917 [ilog2(VM_MAYEXEC)] = "me", 918 [ilog2(VM_MAYSHARE)] = "ms", 919 [ilog2(VM_GROWSDOWN)] = "gd", 920 [ilog2(VM_PFNMAP)] = "pf", 921 [ilog2(VM_LOCKED)] = "lo", 922 [ilog2(VM_IO)] = "io", 923 [ilog2(VM_SEQ_READ)] = "sr", 924 [ilog2(VM_RAND_READ)] = "rr", 925 [ilog2(VM_DONTCOPY)] = "dc", 926 [ilog2(VM_DONTEXPAND)] = "de", 927 [ilog2(VM_LOCKONFAULT)] = "lf", 928 [ilog2(VM_ACCOUNT)] = "ac", 929 [ilog2(VM_NORESERVE)] = "nr", 930 [ilog2(VM_HUGETLB)] = "ht", 931 [ilog2(VM_SYNC)] = "sf", 932 [ilog2(VM_ARCH_1)] = "ar", 933 [ilog2(VM_WIPEONFORK)] = "wf", 934 [ilog2(VM_DONTDUMP)] = "dd", 935 #ifdef CONFIG_ARM64_BTI 936 [ilog2(VM_ARM64_BTI)] = "bt", 937 #endif 938 #ifdef CONFIG_MEM_SOFT_DIRTY 939 [ilog2(VM_SOFTDIRTY)] = "sd", 940 #endif 941 [ilog2(VM_MIXEDMAP)] = "mm", 942 [ilog2(VM_HUGEPAGE)] = "hg", 943 [ilog2(VM_NOHUGEPAGE)] = "nh", 944 [ilog2(VM_MERGEABLE)] = "mg", 945 [ilog2(VM_UFFD_MISSING)]= "um", 946 [ilog2(VM_UFFD_WP)] = "uw", 947 #ifdef CONFIG_ARM64_MTE 948 [ilog2(VM_MTE)] = "mt", 949 [ilog2(VM_MTE_ALLOWED)] = "", 950 #endif 951 #ifdef CONFIG_ARCH_HAS_PKEYS 952 /* These come out via ProtectionKey: */ 953 [ilog2(VM_PKEY_BIT0)] = "", 954 [ilog2(VM_PKEY_BIT1)] = "", 955 [ilog2(VM_PKEY_BIT2)] = "", 956 [ilog2(VM_PKEY_BIT3)] = "", 957 #if VM_PKEY_BIT4 958 [ilog2(VM_PKEY_BIT4)] = "", 959 #endif 960 #endif /* CONFIG_ARCH_HAS_PKEYS */ 961 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 962 [ilog2(VM_UFFD_MINOR)] = "ui", 963 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ 964 #ifdef CONFIG_X86_USER_SHADOW_STACK 965 [ilog2(VM_SHADOW_STACK)] = "ss", 966 #endif 967 #ifdef CONFIG_64BIT 968 [ilog2(VM_SEALED)] = "sl", 969 #endif 970 }; 971 size_t i; 972 973 seq_puts(m, "VmFlags: "); 974 for (i = 0; i < BITS_PER_LONG; i++) { 975 if (!mnemonics[i][0]) 976 continue; 977 if (vma->vm_flags & (1UL << i)) { 978 seq_putc(m, mnemonics[i][0]); 979 seq_putc(m, mnemonics[i][1]); 980 seq_putc(m, ' '); 981 } 982 } 983 seq_putc(m, '\n'); 984 } 985 986 #ifdef CONFIG_HUGETLB_PAGE 987 static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask, 988 unsigned long addr, unsigned long end, 989 struct mm_walk *walk) 990 { 991 struct mem_size_stats *mss = walk->private; 992 struct vm_area_struct *vma = walk->vma; 993 pte_t ptent = huge_ptep_get(pte); 994 struct folio *folio = NULL; 995 bool present = false; 996 997 if (pte_present(ptent)) { 998 folio = page_folio(pte_page(ptent)); 999 present = true; 1000 } else if (is_swap_pte(ptent)) { 1001 swp_entry_t swpent = pte_to_swp_entry(ptent); 1002 1003 if (is_pfn_swap_entry(swpent)) 1004 folio = pfn_swap_entry_folio(swpent); 1005 } 1006 1007 if (folio) { 1008 /* We treat non-present entries as "maybe shared". */ 1009 if (!present || folio_likely_mapped_shared(folio) || 1010 hugetlb_pmd_shared(pte)) 1011 mss->shared_hugetlb += huge_page_size(hstate_vma(vma)); 1012 else 1013 mss->private_hugetlb += huge_page_size(hstate_vma(vma)); 1014 } 1015 return 0; 1016 } 1017 #else 1018 #define smaps_hugetlb_range NULL 1019 #endif /* HUGETLB_PAGE */ 1020 1021 static const struct mm_walk_ops smaps_walk_ops = { 1022 .pmd_entry = smaps_pte_range, 1023 .hugetlb_entry = smaps_hugetlb_range, 1024 .walk_lock = PGWALK_RDLOCK, 1025 }; 1026 1027 static const struct mm_walk_ops smaps_shmem_walk_ops = { 1028 .pmd_entry = smaps_pte_range, 1029 .hugetlb_entry = smaps_hugetlb_range, 1030 .pte_hole = smaps_pte_hole, 1031 .walk_lock = PGWALK_RDLOCK, 1032 }; 1033 1034 /* 1035 * Gather mem stats from @vma with the indicated beginning 1036 * address @start, and keep them in @mss. 1037 * 1038 * Use vm_start of @vma as the beginning address if @start is 0. 1039 */ 1040 static void smap_gather_stats(struct vm_area_struct *vma, 1041 struct mem_size_stats *mss, unsigned long start) 1042 { 1043 const struct mm_walk_ops *ops = &smaps_walk_ops; 1044 1045 /* Invalid start */ 1046 if (start >= vma->vm_end) 1047 return; 1048 1049 if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { 1050 /* 1051 * For shared or readonly shmem mappings we know that all 1052 * swapped out pages belong to the shmem object, and we can 1053 * obtain the swap value much more efficiently. For private 1054 * writable mappings, we might have COW pages that are 1055 * not affected by the parent swapped out pages of the shmem 1056 * object, so we have to distinguish them during the page walk. 1057 * Unless we know that the shmem object (or the part mapped by 1058 * our VMA) has no swapped out pages at all. 1059 */ 1060 unsigned long shmem_swapped = shmem_swap_usage(vma); 1061 1062 if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) || 1063 !(vma->vm_flags & VM_WRITE))) { 1064 mss->swap += shmem_swapped; 1065 } else { 1066 ops = &smaps_shmem_walk_ops; 1067 } 1068 } 1069 1070 /* mmap_lock is held in m_start */ 1071 if (!start) 1072 walk_page_vma(vma, ops, mss); 1073 else 1074 walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss); 1075 } 1076 1077 #define SEQ_PUT_DEC(str, val) \ 1078 seq_put_decimal_ull_width(m, str, (val) >> 10, 8) 1079 1080 /* Show the contents common for smaps and smaps_rollup */ 1081 static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss, 1082 bool rollup_mode) 1083 { 1084 SEQ_PUT_DEC("Rss: ", mss->resident); 1085 SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT); 1086 SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT); 1087 if (rollup_mode) { 1088 /* 1089 * These are meaningful only for smaps_rollup, otherwise two of 1090 * them are zero, and the other one is the same as Pss. 1091 */ 1092 SEQ_PUT_DEC(" kB\nPss_Anon: ", 1093 mss->pss_anon >> PSS_SHIFT); 1094 SEQ_PUT_DEC(" kB\nPss_File: ", 1095 mss->pss_file >> PSS_SHIFT); 1096 SEQ_PUT_DEC(" kB\nPss_Shmem: ", 1097 mss->pss_shmem >> PSS_SHIFT); 1098 } 1099 SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean); 1100 SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty); 1101 SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean); 1102 SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty); 1103 SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced); 1104 SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous); 1105 SEQ_PUT_DEC(" kB\nKSM: ", mss->ksm); 1106 SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree); 1107 SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp); 1108 SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp); 1109 SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp); 1110 SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb); 1111 seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ", 1112 mss->private_hugetlb >> 10, 7); 1113 SEQ_PUT_DEC(" kB\nSwap: ", mss->swap); 1114 SEQ_PUT_DEC(" kB\nSwapPss: ", 1115 mss->swap_pss >> PSS_SHIFT); 1116 SEQ_PUT_DEC(" kB\nLocked: ", 1117 mss->pss_locked >> PSS_SHIFT); 1118 seq_puts(m, " kB\n"); 1119 } 1120 1121 static int show_smap(struct seq_file *m, void *v) 1122 { 1123 struct vm_area_struct *vma = v; 1124 struct mem_size_stats mss = {}; 1125 1126 smap_gather_stats(vma, &mss, 0); 1127 1128 show_map_vma(m, vma); 1129 1130 SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start); 1131 SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma)); 1132 SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma)); 1133 seq_puts(m, " kB\n"); 1134 1135 __show_smap(m, &mss, false); 1136 1137 seq_printf(m, "THPeligible: %8u\n", 1138 !!thp_vma_allowable_orders(vma, vma->vm_flags, 1139 TVA_SMAPS | TVA_ENFORCE_SYSFS, THP_ORDERS_ALL)); 1140 1141 if (arch_pkeys_enabled()) 1142 seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); 1143 show_smap_vma_flags(m, vma); 1144 1145 return 0; 1146 } 1147 1148 static int show_smaps_rollup(struct seq_file *m, void *v) 1149 { 1150 struct proc_maps_private *priv = m->private; 1151 struct mem_size_stats mss = {}; 1152 struct mm_struct *mm = priv->mm; 1153 struct vm_area_struct *vma; 1154 unsigned long vma_start = 0, last_vma_end = 0; 1155 int ret = 0; 1156 VMA_ITERATOR(vmi, mm, 0); 1157 1158 priv->task = get_proc_task(priv->inode); 1159 if (!priv->task) 1160 return -ESRCH; 1161 1162 if (!mm || !mmget_not_zero(mm)) { 1163 ret = -ESRCH; 1164 goto out_put_task; 1165 } 1166 1167 ret = mmap_read_lock_killable(mm); 1168 if (ret) 1169 goto out_put_mm; 1170 1171 hold_task_mempolicy(priv); 1172 vma = vma_next(&vmi); 1173 1174 if (unlikely(!vma)) 1175 goto empty_set; 1176 1177 vma_start = vma->vm_start; 1178 do { 1179 smap_gather_stats(vma, &mss, 0); 1180 last_vma_end = vma->vm_end; 1181 1182 /* 1183 * Release mmap_lock temporarily if someone wants to 1184 * access it for write request. 1185 */ 1186 if (mmap_lock_is_contended(mm)) { 1187 vma_iter_invalidate(&vmi); 1188 mmap_read_unlock(mm); 1189 ret = mmap_read_lock_killable(mm); 1190 if (ret) { 1191 release_task_mempolicy(priv); 1192 goto out_put_mm; 1193 } 1194 1195 /* 1196 * After dropping the lock, there are four cases to 1197 * consider. See the following example for explanation. 1198 * 1199 * +------+------+-----------+ 1200 * | VMA1 | VMA2 | VMA3 | 1201 * +------+------+-----------+ 1202 * | | | | 1203 * 4k 8k 16k 400k 1204 * 1205 * Suppose we drop the lock after reading VMA2 due to 1206 * contention, then we get: 1207 * 1208 * last_vma_end = 16k 1209 * 1210 * 1) VMA2 is freed, but VMA3 exists: 1211 * 1212 * vma_next(vmi) will return VMA3. 1213 * In this case, just continue from VMA3. 1214 * 1215 * 2) VMA2 still exists: 1216 * 1217 * vma_next(vmi) will return VMA3. 1218 * In this case, just continue from VMA3. 1219 * 1220 * 3) No more VMAs can be found: 1221 * 1222 * vma_next(vmi) will return NULL. 1223 * No more things to do, just break. 1224 * 1225 * 4) (last_vma_end - 1) is the middle of a vma (VMA'): 1226 * 1227 * vma_next(vmi) will return VMA' whose range 1228 * contains last_vma_end. 1229 * Iterate VMA' from last_vma_end. 1230 */ 1231 vma = vma_next(&vmi); 1232 /* Case 3 above */ 1233 if (!vma) 1234 break; 1235 1236 /* Case 1 and 2 above */ 1237 if (vma->vm_start >= last_vma_end) { 1238 smap_gather_stats(vma, &mss, 0); 1239 last_vma_end = vma->vm_end; 1240 continue; 1241 } 1242 1243 /* Case 4 above */ 1244 if (vma->vm_end > last_vma_end) { 1245 smap_gather_stats(vma, &mss, last_vma_end); 1246 last_vma_end = vma->vm_end; 1247 } 1248 } 1249 } for_each_vma(vmi, vma); 1250 1251 empty_set: 1252 show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); 1253 seq_pad(m, ' '); 1254 seq_puts(m, "[rollup]\n"); 1255 1256 __show_smap(m, &mss, true); 1257 1258 release_task_mempolicy(priv); 1259 mmap_read_unlock(mm); 1260 1261 out_put_mm: 1262 mmput(mm); 1263 out_put_task: 1264 put_task_struct(priv->task); 1265 priv->task = NULL; 1266 1267 return ret; 1268 } 1269 #undef SEQ_PUT_DEC 1270 1271 static const struct seq_operations proc_pid_smaps_op = { 1272 .start = m_start, 1273 .next = m_next, 1274 .stop = m_stop, 1275 .show = show_smap 1276 }; 1277 1278 static int pid_smaps_open(struct inode *inode, struct file *file) 1279 { 1280 return do_maps_open(inode, file, &proc_pid_smaps_op); 1281 } 1282 1283 static int smaps_rollup_open(struct inode *inode, struct file *file) 1284 { 1285 int ret; 1286 struct proc_maps_private *priv; 1287 1288 priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT); 1289 if (!priv) 1290 return -ENOMEM; 1291 1292 ret = single_open(file, show_smaps_rollup, priv); 1293 if (ret) 1294 goto out_free; 1295 1296 priv->inode = inode; 1297 priv->mm = proc_mem_open(inode, PTRACE_MODE_READ); 1298 if (IS_ERR(priv->mm)) { 1299 ret = PTR_ERR(priv->mm); 1300 1301 single_release(inode, file); 1302 goto out_free; 1303 } 1304 1305 return 0; 1306 1307 out_free: 1308 kfree(priv); 1309 return ret; 1310 } 1311 1312 static int smaps_rollup_release(struct inode *inode, struct file *file) 1313 { 1314 struct seq_file *seq = file->private_data; 1315 struct proc_maps_private *priv = seq->private; 1316 1317 if (priv->mm) 1318 mmdrop(priv->mm); 1319 1320 kfree(priv); 1321 return single_release(inode, file); 1322 } 1323 1324 const struct file_operations proc_pid_smaps_operations = { 1325 .open = pid_smaps_open, 1326 .read = seq_read, 1327 .llseek = seq_lseek, 1328 .release = proc_map_release, 1329 }; 1330 1331 const struct file_operations proc_pid_smaps_rollup_operations = { 1332 .open = smaps_rollup_open, 1333 .read = seq_read, 1334 .llseek = seq_lseek, 1335 .release = smaps_rollup_release, 1336 }; 1337 1338 enum clear_refs_types { 1339 CLEAR_REFS_ALL = 1, 1340 CLEAR_REFS_ANON, 1341 CLEAR_REFS_MAPPED, 1342 CLEAR_REFS_SOFT_DIRTY, 1343 CLEAR_REFS_MM_HIWATER_RSS, 1344 CLEAR_REFS_LAST, 1345 }; 1346 1347 struct clear_refs_private { 1348 enum clear_refs_types type; 1349 }; 1350 1351 #ifdef CONFIG_MEM_SOFT_DIRTY 1352 1353 static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) 1354 { 1355 struct folio *folio; 1356 1357 if (!pte_write(pte)) 1358 return false; 1359 if (!is_cow_mapping(vma->vm_flags)) 1360 return false; 1361 if (likely(!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))) 1362 return false; 1363 folio = vm_normal_folio(vma, addr, pte); 1364 if (!folio) 1365 return false; 1366 return folio_maybe_dma_pinned(folio); 1367 } 1368 1369 static inline void clear_soft_dirty(struct vm_area_struct *vma, 1370 unsigned long addr, pte_t *pte) 1371 { 1372 /* 1373 * The soft-dirty tracker uses #PF-s to catch writes 1374 * to pages, so write-protect the pte as well. See the 1375 * Documentation/admin-guide/mm/soft-dirty.rst for full description 1376 * of how soft-dirty works. 1377 */ 1378 pte_t ptent = ptep_get(pte); 1379 1380 if (pte_present(ptent)) { 1381 pte_t old_pte; 1382 1383 if (pte_is_pinned(vma, addr, ptent)) 1384 return; 1385 old_pte = ptep_modify_prot_start(vma, addr, pte); 1386 ptent = pte_wrprotect(old_pte); 1387 ptent = pte_clear_soft_dirty(ptent); 1388 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); 1389 } else if (is_swap_pte(ptent)) { 1390 ptent = pte_swp_clear_soft_dirty(ptent); 1391 set_pte_at(vma->vm_mm, addr, pte, ptent); 1392 } 1393 } 1394 #else 1395 static inline void clear_soft_dirty(struct vm_area_struct *vma, 1396 unsigned long addr, pte_t *pte) 1397 { 1398 } 1399 #endif 1400 1401 #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE) 1402 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, 1403 unsigned long addr, pmd_t *pmdp) 1404 { 1405 pmd_t old, pmd = *pmdp; 1406 1407 if (pmd_present(pmd)) { 1408 /* See comment in change_huge_pmd() */ 1409 old = pmdp_invalidate(vma, addr, pmdp); 1410 if (pmd_dirty(old)) 1411 pmd = pmd_mkdirty(pmd); 1412 if (pmd_young(old)) 1413 pmd = pmd_mkyoung(pmd); 1414 1415 pmd = pmd_wrprotect(pmd); 1416 pmd = pmd_clear_soft_dirty(pmd); 1417 1418 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 1419 } else if (is_migration_entry(pmd_to_swp_entry(pmd))) { 1420 pmd = pmd_swp_clear_soft_dirty(pmd); 1421 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 1422 } 1423 } 1424 #else 1425 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, 1426 unsigned long addr, pmd_t *pmdp) 1427 { 1428 } 1429 #endif 1430 1431 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, 1432 unsigned long end, struct mm_walk *walk) 1433 { 1434 struct clear_refs_private *cp = walk->private; 1435 struct vm_area_struct *vma = walk->vma; 1436 pte_t *pte, ptent; 1437 spinlock_t *ptl; 1438 struct folio *folio; 1439 1440 ptl = pmd_trans_huge_lock(pmd, vma); 1441 if (ptl) { 1442 if (cp->type == CLEAR_REFS_SOFT_DIRTY) { 1443 clear_soft_dirty_pmd(vma, addr, pmd); 1444 goto out; 1445 } 1446 1447 if (!pmd_present(*pmd)) 1448 goto out; 1449 1450 folio = pmd_folio(*pmd); 1451 1452 /* Clear accessed and referenced bits. */ 1453 pmdp_test_and_clear_young(vma, addr, pmd); 1454 folio_test_clear_young(folio); 1455 folio_clear_referenced(folio); 1456 out: 1457 spin_unlock(ptl); 1458 return 0; 1459 } 1460 1461 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1462 if (!pte) { 1463 walk->action = ACTION_AGAIN; 1464 return 0; 1465 } 1466 for (; addr != end; pte++, addr += PAGE_SIZE) { 1467 ptent = ptep_get(pte); 1468 1469 if (cp->type == CLEAR_REFS_SOFT_DIRTY) { 1470 clear_soft_dirty(vma, addr, pte); 1471 continue; 1472 } 1473 1474 if (!pte_present(ptent)) 1475 continue; 1476 1477 folio = vm_normal_folio(vma, addr, ptent); 1478 if (!folio) 1479 continue; 1480 1481 /* Clear accessed and referenced bits. */ 1482 ptep_test_and_clear_young(vma, addr, pte); 1483 folio_test_clear_young(folio); 1484 folio_clear_referenced(folio); 1485 } 1486 pte_unmap_unlock(pte - 1, ptl); 1487 cond_resched(); 1488 return 0; 1489 } 1490 1491 static int clear_refs_test_walk(unsigned long start, unsigned long end, 1492 struct mm_walk *walk) 1493 { 1494 struct clear_refs_private *cp = walk->private; 1495 struct vm_area_struct *vma = walk->vma; 1496 1497 if (vma->vm_flags & VM_PFNMAP) 1498 return 1; 1499 1500 /* 1501 * Writing 1 to /proc/pid/clear_refs affects all pages. 1502 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages. 1503 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages. 1504 * Writing 4 to /proc/pid/clear_refs affects all pages. 1505 */ 1506 if (cp->type == CLEAR_REFS_ANON && vma->vm_file) 1507 return 1; 1508 if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file) 1509 return 1; 1510 return 0; 1511 } 1512 1513 static const struct mm_walk_ops clear_refs_walk_ops = { 1514 .pmd_entry = clear_refs_pte_range, 1515 .test_walk = clear_refs_test_walk, 1516 .walk_lock = PGWALK_WRLOCK, 1517 }; 1518 1519 static ssize_t clear_refs_write(struct file *file, const char __user *buf, 1520 size_t count, loff_t *ppos) 1521 { 1522 struct task_struct *task; 1523 char buffer[PROC_NUMBUF] = {}; 1524 struct mm_struct *mm; 1525 struct vm_area_struct *vma; 1526 enum clear_refs_types type; 1527 int itype; 1528 int rv; 1529 1530 if (count > sizeof(buffer) - 1) 1531 count = sizeof(buffer) - 1; 1532 if (copy_from_user(buffer, buf, count)) 1533 return -EFAULT; 1534 rv = kstrtoint(strstrip(buffer), 10, &itype); 1535 if (rv < 0) 1536 return rv; 1537 type = (enum clear_refs_types)itype; 1538 if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST) 1539 return -EINVAL; 1540 1541 task = get_proc_task(file_inode(file)); 1542 if (!task) 1543 return -ESRCH; 1544 mm = get_task_mm(task); 1545 if (mm) { 1546 VMA_ITERATOR(vmi, mm, 0); 1547 struct mmu_notifier_range range; 1548 struct clear_refs_private cp = { 1549 .type = type, 1550 }; 1551 1552 if (mmap_write_lock_killable(mm)) { 1553 count = -EINTR; 1554 goto out_mm; 1555 } 1556 if (type == CLEAR_REFS_MM_HIWATER_RSS) { 1557 /* 1558 * Writing 5 to /proc/pid/clear_refs resets the peak 1559 * resident set size to this mm's current rss value. 1560 */ 1561 reset_mm_hiwater_rss(mm); 1562 goto out_unlock; 1563 } 1564 1565 if (type == CLEAR_REFS_SOFT_DIRTY) { 1566 for_each_vma(vmi, vma) { 1567 if (!(vma->vm_flags & VM_SOFTDIRTY)) 1568 continue; 1569 vm_flags_clear(vma, VM_SOFTDIRTY); 1570 vma_set_page_prot(vma); 1571 } 1572 1573 inc_tlb_flush_pending(mm); 1574 mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY, 1575 0, mm, 0, -1UL); 1576 mmu_notifier_invalidate_range_start(&range); 1577 } 1578 walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp); 1579 if (type == CLEAR_REFS_SOFT_DIRTY) { 1580 mmu_notifier_invalidate_range_end(&range); 1581 flush_tlb_mm(mm); 1582 dec_tlb_flush_pending(mm); 1583 } 1584 out_unlock: 1585 mmap_write_unlock(mm); 1586 out_mm: 1587 mmput(mm); 1588 } 1589 put_task_struct(task); 1590 1591 return count; 1592 } 1593 1594 const struct file_operations proc_clear_refs_operations = { 1595 .write = clear_refs_write, 1596 .llseek = noop_llseek, 1597 }; 1598 1599 typedef struct { 1600 u64 pme; 1601 } pagemap_entry_t; 1602 1603 struct pagemapread { 1604 int pos, len; /* units: PM_ENTRY_BYTES, not bytes */ 1605 pagemap_entry_t *buffer; 1606 bool show_pfn; 1607 }; 1608 1609 #define PAGEMAP_WALK_SIZE (PMD_SIZE) 1610 #define PAGEMAP_WALK_MASK (PMD_MASK) 1611 1612 #define PM_ENTRY_BYTES sizeof(pagemap_entry_t) 1613 #define PM_PFRAME_BITS 55 1614 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) 1615 #define PM_SOFT_DIRTY BIT_ULL(55) 1616 #define PM_MMAP_EXCLUSIVE BIT_ULL(56) 1617 #define PM_UFFD_WP BIT_ULL(57) 1618 #define PM_FILE BIT_ULL(61) 1619 #define PM_SWAP BIT_ULL(62) 1620 #define PM_PRESENT BIT_ULL(63) 1621 1622 #define PM_END_OF_BUFFER 1 1623 1624 static inline pagemap_entry_t make_pme(u64 frame, u64 flags) 1625 { 1626 return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags }; 1627 } 1628 1629 static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) 1630 { 1631 pm->buffer[pm->pos++] = *pme; 1632 if (pm->pos >= pm->len) 1633 return PM_END_OF_BUFFER; 1634 return 0; 1635 } 1636 1637 static int pagemap_pte_hole(unsigned long start, unsigned long end, 1638 __always_unused int depth, struct mm_walk *walk) 1639 { 1640 struct pagemapread *pm = walk->private; 1641 unsigned long addr = start; 1642 int err = 0; 1643 1644 while (addr < end) { 1645 struct vm_area_struct *vma = find_vma(walk->mm, addr); 1646 pagemap_entry_t pme = make_pme(0, 0); 1647 /* End of address space hole, which we mark as non-present. */ 1648 unsigned long hole_end; 1649 1650 if (vma) 1651 hole_end = min(end, vma->vm_start); 1652 else 1653 hole_end = end; 1654 1655 for (; addr < hole_end; addr += PAGE_SIZE) { 1656 err = add_to_pagemap(&pme, pm); 1657 if (err) 1658 goto out; 1659 } 1660 1661 if (!vma) 1662 break; 1663 1664 /* Addresses in the VMA. */ 1665 if (vma->vm_flags & VM_SOFTDIRTY) 1666 pme = make_pme(0, PM_SOFT_DIRTY); 1667 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { 1668 err = add_to_pagemap(&pme, pm); 1669 if (err) 1670 goto out; 1671 } 1672 } 1673 out: 1674 return err; 1675 } 1676 1677 static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, 1678 struct vm_area_struct *vma, unsigned long addr, pte_t pte) 1679 { 1680 u64 frame = 0, flags = 0; 1681 struct page *page = NULL; 1682 struct folio *folio; 1683 1684 if (pte_present(pte)) { 1685 if (pm->show_pfn) 1686 frame = pte_pfn(pte); 1687 flags |= PM_PRESENT; 1688 page = vm_normal_page(vma, addr, pte); 1689 if (pte_soft_dirty(pte)) 1690 flags |= PM_SOFT_DIRTY; 1691 if (pte_uffd_wp(pte)) 1692 flags |= PM_UFFD_WP; 1693 } else if (is_swap_pte(pte)) { 1694 swp_entry_t entry; 1695 if (pte_swp_soft_dirty(pte)) 1696 flags |= PM_SOFT_DIRTY; 1697 if (pte_swp_uffd_wp(pte)) 1698 flags |= PM_UFFD_WP; 1699 entry = pte_to_swp_entry(pte); 1700 if (pm->show_pfn) { 1701 pgoff_t offset; 1702 /* 1703 * For PFN swap offsets, keeping the offset field 1704 * to be PFN only to be compatible with old smaps. 1705 */ 1706 if (is_pfn_swap_entry(entry)) 1707 offset = swp_offset_pfn(entry); 1708 else 1709 offset = swp_offset(entry); 1710 frame = swp_type(entry) | 1711 (offset << MAX_SWAPFILES_SHIFT); 1712 } 1713 flags |= PM_SWAP; 1714 if (is_pfn_swap_entry(entry)) 1715 page = pfn_swap_entry_to_page(entry); 1716 if (pte_marker_entry_uffd_wp(entry)) 1717 flags |= PM_UFFD_WP; 1718 } 1719 1720 if (page) { 1721 folio = page_folio(page); 1722 if (!folio_test_anon(folio)) 1723 flags |= PM_FILE; 1724 if ((flags & PM_PRESENT) && 1725 folio_precise_page_mapcount(folio, page) == 1) 1726 flags |= PM_MMAP_EXCLUSIVE; 1727 } 1728 if (vma->vm_flags & VM_SOFTDIRTY) 1729 flags |= PM_SOFT_DIRTY; 1730 1731 return make_pme(frame, flags); 1732 } 1733 1734 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, 1735 struct mm_walk *walk) 1736 { 1737 struct vm_area_struct *vma = walk->vma; 1738 struct pagemapread *pm = walk->private; 1739 spinlock_t *ptl; 1740 pte_t *pte, *orig_pte; 1741 int err = 0; 1742 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1743 1744 ptl = pmd_trans_huge_lock(pmdp, vma); 1745 if (ptl) { 1746 unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; 1747 u64 flags = 0, frame = 0; 1748 pmd_t pmd = *pmdp; 1749 struct page *page = NULL; 1750 struct folio *folio = NULL; 1751 1752 if (vma->vm_flags & VM_SOFTDIRTY) 1753 flags |= PM_SOFT_DIRTY; 1754 1755 if (pmd_present(pmd)) { 1756 page = pmd_page(pmd); 1757 1758 flags |= PM_PRESENT; 1759 if (pmd_soft_dirty(pmd)) 1760 flags |= PM_SOFT_DIRTY; 1761 if (pmd_uffd_wp(pmd)) 1762 flags |= PM_UFFD_WP; 1763 if (pm->show_pfn) 1764 frame = pmd_pfn(pmd) + idx; 1765 } 1766 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1767 else if (is_swap_pmd(pmd)) { 1768 swp_entry_t entry = pmd_to_swp_entry(pmd); 1769 unsigned long offset; 1770 1771 if (pm->show_pfn) { 1772 if (is_pfn_swap_entry(entry)) 1773 offset = swp_offset_pfn(entry) + idx; 1774 else 1775 offset = swp_offset(entry) + idx; 1776 frame = swp_type(entry) | 1777 (offset << MAX_SWAPFILES_SHIFT); 1778 } 1779 flags |= PM_SWAP; 1780 if (pmd_swp_soft_dirty(pmd)) 1781 flags |= PM_SOFT_DIRTY; 1782 if (pmd_swp_uffd_wp(pmd)) 1783 flags |= PM_UFFD_WP; 1784 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1785 page = pfn_swap_entry_to_page(entry); 1786 } 1787 #endif 1788 1789 if (page) { 1790 folio = page_folio(page); 1791 if (!folio_test_anon(folio)) 1792 flags |= PM_FILE; 1793 } 1794 1795 for (; addr != end; addr += PAGE_SIZE, idx++) { 1796 unsigned long cur_flags = flags; 1797 pagemap_entry_t pme; 1798 1799 if (folio && (flags & PM_PRESENT) && 1800 folio_precise_page_mapcount(folio, page + idx) == 1) 1801 cur_flags |= PM_MMAP_EXCLUSIVE; 1802 1803 pme = make_pme(frame, cur_flags); 1804 err = add_to_pagemap(&pme, pm); 1805 if (err) 1806 break; 1807 if (pm->show_pfn) { 1808 if (flags & PM_PRESENT) 1809 frame++; 1810 else if (flags & PM_SWAP) 1811 frame += (1 << MAX_SWAPFILES_SHIFT); 1812 } 1813 } 1814 spin_unlock(ptl); 1815 return err; 1816 } 1817 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1818 1819 /* 1820 * We can assume that @vma always points to a valid one and @end never 1821 * goes beyond vma->vm_end. 1822 */ 1823 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl); 1824 if (!pte) { 1825 walk->action = ACTION_AGAIN; 1826 return err; 1827 } 1828 for (; addr < end; pte++, addr += PAGE_SIZE) { 1829 pagemap_entry_t pme; 1830 1831 pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); 1832 err = add_to_pagemap(&pme, pm); 1833 if (err) 1834 break; 1835 } 1836 pte_unmap_unlock(orig_pte, ptl); 1837 1838 cond_resched(); 1839 1840 return err; 1841 } 1842 1843 #ifdef CONFIG_HUGETLB_PAGE 1844 /* This function walks within one hugetlb entry in the single call */ 1845 static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask, 1846 unsigned long addr, unsigned long end, 1847 struct mm_walk *walk) 1848 { 1849 struct pagemapread *pm = walk->private; 1850 struct vm_area_struct *vma = walk->vma; 1851 u64 flags = 0, frame = 0; 1852 int err = 0; 1853 pte_t pte; 1854 1855 if (vma->vm_flags & VM_SOFTDIRTY) 1856 flags |= PM_SOFT_DIRTY; 1857 1858 pte = huge_ptep_get(ptep); 1859 if (pte_present(pte)) { 1860 struct folio *folio = page_folio(pte_page(pte)); 1861 1862 if (!folio_test_anon(folio)) 1863 flags |= PM_FILE; 1864 1865 if (!folio_likely_mapped_shared(folio) && 1866 !hugetlb_pmd_shared(ptep)) 1867 flags |= PM_MMAP_EXCLUSIVE; 1868 1869 if (huge_pte_uffd_wp(pte)) 1870 flags |= PM_UFFD_WP; 1871 1872 flags |= PM_PRESENT; 1873 if (pm->show_pfn) 1874 frame = pte_pfn(pte) + 1875 ((addr & ~hmask) >> PAGE_SHIFT); 1876 } else if (pte_swp_uffd_wp_any(pte)) { 1877 flags |= PM_UFFD_WP; 1878 } 1879 1880 for (; addr != end; addr += PAGE_SIZE) { 1881 pagemap_entry_t pme = make_pme(frame, flags); 1882 1883 err = add_to_pagemap(&pme, pm); 1884 if (err) 1885 return err; 1886 if (pm->show_pfn && (flags & PM_PRESENT)) 1887 frame++; 1888 } 1889 1890 cond_resched(); 1891 1892 return err; 1893 } 1894 #else 1895 #define pagemap_hugetlb_range NULL 1896 #endif /* HUGETLB_PAGE */ 1897 1898 static const struct mm_walk_ops pagemap_ops = { 1899 .pmd_entry = pagemap_pmd_range, 1900 .pte_hole = pagemap_pte_hole, 1901 .hugetlb_entry = pagemap_hugetlb_range, 1902 .walk_lock = PGWALK_RDLOCK, 1903 }; 1904 1905 /* 1906 * /proc/pid/pagemap - an array mapping virtual pages to pfns 1907 * 1908 * For each page in the address space, this file contains one 64-bit entry 1909 * consisting of the following: 1910 * 1911 * Bits 0-54 page frame number (PFN) if present 1912 * Bits 0-4 swap type if swapped 1913 * Bits 5-54 swap offset if swapped 1914 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst) 1915 * Bit 56 page exclusively mapped 1916 * Bit 57 pte is uffd-wp write-protected 1917 * Bits 58-60 zero 1918 * Bit 61 page is file-page or shared-anon 1919 * Bit 62 page swapped 1920 * Bit 63 page present 1921 * 1922 * If the page is not present but in swap, then the PFN contains an 1923 * encoding of the swap file number and the page's offset into the 1924 * swap. Unmapped pages return a null PFN. This allows determining 1925 * precisely which pages are mapped (or in swap) and comparing mapped 1926 * pages between processes. 1927 * 1928 * Efficient users of this interface will use /proc/pid/maps to 1929 * determine which areas of memory are actually mapped and llseek to 1930 * skip over unmapped regions. 1931 */ 1932 static ssize_t pagemap_read(struct file *file, char __user *buf, 1933 size_t count, loff_t *ppos) 1934 { 1935 struct mm_struct *mm = file->private_data; 1936 struct pagemapread pm; 1937 unsigned long src; 1938 unsigned long svpfn; 1939 unsigned long start_vaddr; 1940 unsigned long end_vaddr; 1941 int ret = 0, copied = 0; 1942 1943 if (!mm || !mmget_not_zero(mm)) 1944 goto out; 1945 1946 ret = -EINVAL; 1947 /* file position must be aligned */ 1948 if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES)) 1949 goto out_mm; 1950 1951 ret = 0; 1952 if (!count) 1953 goto out_mm; 1954 1955 /* do not disclose physical addresses: attack vector */ 1956 pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); 1957 1958 pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); 1959 pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); 1960 ret = -ENOMEM; 1961 if (!pm.buffer) 1962 goto out_mm; 1963 1964 src = *ppos; 1965 svpfn = src / PM_ENTRY_BYTES; 1966 end_vaddr = mm->task_size; 1967 1968 /* watch out for wraparound */ 1969 start_vaddr = end_vaddr; 1970 if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { 1971 unsigned long end; 1972 1973 ret = mmap_read_lock_killable(mm); 1974 if (ret) 1975 goto out_free; 1976 start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); 1977 mmap_read_unlock(mm); 1978 1979 end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); 1980 if (end >= start_vaddr && end < mm->task_size) 1981 end_vaddr = end; 1982 } 1983 1984 /* Ensure the address is inside the task */ 1985 if (start_vaddr > mm->task_size) 1986 start_vaddr = end_vaddr; 1987 1988 ret = 0; 1989 while (count && (start_vaddr < end_vaddr)) { 1990 int len; 1991 unsigned long end; 1992 1993 pm.pos = 0; 1994 end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; 1995 /* overflow ? */ 1996 if (end < start_vaddr || end > end_vaddr) 1997 end = end_vaddr; 1998 ret = mmap_read_lock_killable(mm); 1999 if (ret) 2000 goto out_free; 2001 ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm); 2002 mmap_read_unlock(mm); 2003 start_vaddr = end; 2004 2005 len = min(count, PM_ENTRY_BYTES * pm.pos); 2006 if (copy_to_user(buf, pm.buffer, len)) { 2007 ret = -EFAULT; 2008 goto out_free; 2009 } 2010 copied += len; 2011 buf += len; 2012 count -= len; 2013 } 2014 *ppos += copied; 2015 if (!ret || ret == PM_END_OF_BUFFER) 2016 ret = copied; 2017 2018 out_free: 2019 kfree(pm.buffer); 2020 out_mm: 2021 mmput(mm); 2022 out: 2023 return ret; 2024 } 2025 2026 static int pagemap_open(struct inode *inode, struct file *file) 2027 { 2028 struct mm_struct *mm; 2029 2030 mm = proc_mem_open(inode, PTRACE_MODE_READ); 2031 if (IS_ERR(mm)) 2032 return PTR_ERR(mm); 2033 file->private_data = mm; 2034 return 0; 2035 } 2036 2037 static int pagemap_release(struct inode *inode, struct file *file) 2038 { 2039 struct mm_struct *mm = file->private_data; 2040 2041 if (mm) 2042 mmdrop(mm); 2043 return 0; 2044 } 2045 2046 #define PM_SCAN_CATEGORIES (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ 2047 PAGE_IS_FILE | PAGE_IS_PRESENT | \ 2048 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ 2049 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY) 2050 #define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC) 2051 2052 struct pagemap_scan_private { 2053 struct pm_scan_arg arg; 2054 unsigned long masks_of_interest, cur_vma_category; 2055 struct page_region *vec_buf; 2056 unsigned long vec_buf_len, vec_buf_index, found_pages; 2057 struct page_region __user *vec_out; 2058 }; 2059 2060 static unsigned long pagemap_page_category(struct pagemap_scan_private *p, 2061 struct vm_area_struct *vma, 2062 unsigned long addr, pte_t pte) 2063 { 2064 unsigned long categories = 0; 2065 2066 if (pte_present(pte)) { 2067 struct page *page; 2068 2069 categories |= PAGE_IS_PRESENT; 2070 if (!pte_uffd_wp(pte)) 2071 categories |= PAGE_IS_WRITTEN; 2072 2073 if (p->masks_of_interest & PAGE_IS_FILE) { 2074 page = vm_normal_page(vma, addr, pte); 2075 if (page && !PageAnon(page)) 2076 categories |= PAGE_IS_FILE; 2077 } 2078 2079 if (is_zero_pfn(pte_pfn(pte))) 2080 categories |= PAGE_IS_PFNZERO; 2081 if (pte_soft_dirty(pte)) 2082 categories |= PAGE_IS_SOFT_DIRTY; 2083 } else if (is_swap_pte(pte)) { 2084 swp_entry_t swp; 2085 2086 categories |= PAGE_IS_SWAPPED; 2087 if (!pte_swp_uffd_wp_any(pte)) 2088 categories |= PAGE_IS_WRITTEN; 2089 2090 if (p->masks_of_interest & PAGE_IS_FILE) { 2091 swp = pte_to_swp_entry(pte); 2092 if (is_pfn_swap_entry(swp) && 2093 !folio_test_anon(pfn_swap_entry_folio(swp))) 2094 categories |= PAGE_IS_FILE; 2095 } 2096 if (pte_swp_soft_dirty(pte)) 2097 categories |= PAGE_IS_SOFT_DIRTY; 2098 } 2099 2100 return categories; 2101 } 2102 2103 static void make_uffd_wp_pte(struct vm_area_struct *vma, 2104 unsigned long addr, pte_t *pte, pte_t ptent) 2105 { 2106 if (pte_present(ptent)) { 2107 pte_t old_pte; 2108 2109 old_pte = ptep_modify_prot_start(vma, addr, pte); 2110 ptent = pte_mkuffd_wp(old_pte); 2111 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); 2112 } else if (is_swap_pte(ptent)) { 2113 ptent = pte_swp_mkuffd_wp(ptent); 2114 set_pte_at(vma->vm_mm, addr, pte, ptent); 2115 } else { 2116 set_pte_at(vma->vm_mm, addr, pte, 2117 make_pte_marker(PTE_MARKER_UFFD_WP)); 2118 } 2119 } 2120 2121 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2122 static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, 2123 struct vm_area_struct *vma, 2124 unsigned long addr, pmd_t pmd) 2125 { 2126 unsigned long categories = PAGE_IS_HUGE; 2127 2128 if (pmd_present(pmd)) { 2129 struct page *page; 2130 2131 categories |= PAGE_IS_PRESENT; 2132 if (!pmd_uffd_wp(pmd)) 2133 categories |= PAGE_IS_WRITTEN; 2134 2135 if (p->masks_of_interest & PAGE_IS_FILE) { 2136 page = vm_normal_page_pmd(vma, addr, pmd); 2137 if (page && !PageAnon(page)) 2138 categories |= PAGE_IS_FILE; 2139 } 2140 2141 if (is_zero_pfn(pmd_pfn(pmd))) 2142 categories |= PAGE_IS_PFNZERO; 2143 if (pmd_soft_dirty(pmd)) 2144 categories |= PAGE_IS_SOFT_DIRTY; 2145 } else if (is_swap_pmd(pmd)) { 2146 swp_entry_t swp; 2147 2148 categories |= PAGE_IS_SWAPPED; 2149 if (!pmd_swp_uffd_wp(pmd)) 2150 categories |= PAGE_IS_WRITTEN; 2151 if (pmd_swp_soft_dirty(pmd)) 2152 categories |= PAGE_IS_SOFT_DIRTY; 2153 2154 if (p->masks_of_interest & PAGE_IS_FILE) { 2155 swp = pmd_to_swp_entry(pmd); 2156 if (is_pfn_swap_entry(swp) && 2157 !folio_test_anon(pfn_swap_entry_folio(swp))) 2158 categories |= PAGE_IS_FILE; 2159 } 2160 } 2161 2162 return categories; 2163 } 2164 2165 static void make_uffd_wp_pmd(struct vm_area_struct *vma, 2166 unsigned long addr, pmd_t *pmdp) 2167 { 2168 pmd_t old, pmd = *pmdp; 2169 2170 if (pmd_present(pmd)) { 2171 old = pmdp_invalidate_ad(vma, addr, pmdp); 2172 pmd = pmd_mkuffd_wp(old); 2173 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 2174 } else if (is_migration_entry(pmd_to_swp_entry(pmd))) { 2175 pmd = pmd_swp_mkuffd_wp(pmd); 2176 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 2177 } 2178 } 2179 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2180 2181 #ifdef CONFIG_HUGETLB_PAGE 2182 static unsigned long pagemap_hugetlb_category(pte_t pte) 2183 { 2184 unsigned long categories = PAGE_IS_HUGE; 2185 2186 /* 2187 * According to pagemap_hugetlb_range(), file-backed HugeTLB 2188 * page cannot be swapped. So PAGE_IS_FILE is not checked for 2189 * swapped pages. 2190 */ 2191 if (pte_present(pte)) { 2192 categories |= PAGE_IS_PRESENT; 2193 if (!huge_pte_uffd_wp(pte)) 2194 categories |= PAGE_IS_WRITTEN; 2195 if (!PageAnon(pte_page(pte))) 2196 categories |= PAGE_IS_FILE; 2197 if (is_zero_pfn(pte_pfn(pte))) 2198 categories |= PAGE_IS_PFNZERO; 2199 if (pte_soft_dirty(pte)) 2200 categories |= PAGE_IS_SOFT_DIRTY; 2201 } else if (is_swap_pte(pte)) { 2202 categories |= PAGE_IS_SWAPPED; 2203 if (!pte_swp_uffd_wp_any(pte)) 2204 categories |= PAGE_IS_WRITTEN; 2205 if (pte_swp_soft_dirty(pte)) 2206 categories |= PAGE_IS_SOFT_DIRTY; 2207 } 2208 2209 return categories; 2210 } 2211 2212 static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, 2213 unsigned long addr, pte_t *ptep, 2214 pte_t ptent) 2215 { 2216 unsigned long psize; 2217 2218 if (is_hugetlb_entry_hwpoisoned(ptent) || is_pte_marker(ptent)) 2219 return; 2220 2221 psize = huge_page_size(hstate_vma(vma)); 2222 2223 if (is_hugetlb_entry_migration(ptent)) 2224 set_huge_pte_at(vma->vm_mm, addr, ptep, 2225 pte_swp_mkuffd_wp(ptent), psize); 2226 else if (!huge_pte_none(ptent)) 2227 huge_ptep_modify_prot_commit(vma, addr, ptep, ptent, 2228 huge_pte_mkuffd_wp(ptent)); 2229 else 2230 set_huge_pte_at(vma->vm_mm, addr, ptep, 2231 make_pte_marker(PTE_MARKER_UFFD_WP), psize); 2232 } 2233 #endif /* CONFIG_HUGETLB_PAGE */ 2234 2235 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) 2236 static void pagemap_scan_backout_range(struct pagemap_scan_private *p, 2237 unsigned long addr, unsigned long end) 2238 { 2239 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; 2240 2241 if (cur_buf->start != addr) 2242 cur_buf->end = addr; 2243 else 2244 cur_buf->start = cur_buf->end = 0; 2245 2246 p->found_pages -= (end - addr) / PAGE_SIZE; 2247 } 2248 #endif 2249 2250 static bool pagemap_scan_is_interesting_page(unsigned long categories, 2251 const struct pagemap_scan_private *p) 2252 { 2253 categories ^= p->arg.category_inverted; 2254 if ((categories & p->arg.category_mask) != p->arg.category_mask) 2255 return false; 2256 if (p->arg.category_anyof_mask && !(categories & p->arg.category_anyof_mask)) 2257 return false; 2258 2259 return true; 2260 } 2261 2262 static bool pagemap_scan_is_interesting_vma(unsigned long categories, 2263 const struct pagemap_scan_private *p) 2264 { 2265 unsigned long required = p->arg.category_mask & PAGE_IS_WPALLOWED; 2266 2267 categories ^= p->arg.category_inverted; 2268 if ((categories & required) != required) 2269 return false; 2270 2271 return true; 2272 } 2273 2274 static int pagemap_scan_test_walk(unsigned long start, unsigned long end, 2275 struct mm_walk *walk) 2276 { 2277 struct pagemap_scan_private *p = walk->private; 2278 struct vm_area_struct *vma = walk->vma; 2279 unsigned long vma_category = 0; 2280 bool wp_allowed = userfaultfd_wp_async(vma) && 2281 userfaultfd_wp_use_markers(vma); 2282 2283 if (!wp_allowed) { 2284 /* User requested explicit failure over wp-async capability */ 2285 if (p->arg.flags & PM_SCAN_CHECK_WPASYNC) 2286 return -EPERM; 2287 /* 2288 * User requires wr-protect, and allows silently skipping 2289 * unsupported vmas. 2290 */ 2291 if (p->arg.flags & PM_SCAN_WP_MATCHING) 2292 return 1; 2293 /* 2294 * Then the request doesn't involve wr-protects at all, 2295 * fall through to the rest checks, and allow vma walk. 2296 */ 2297 } 2298 2299 if (vma->vm_flags & VM_PFNMAP) 2300 return 1; 2301 2302 if (wp_allowed) 2303 vma_category |= PAGE_IS_WPALLOWED; 2304 2305 if (vma->vm_flags & VM_SOFTDIRTY) 2306 vma_category |= PAGE_IS_SOFT_DIRTY; 2307 2308 if (!pagemap_scan_is_interesting_vma(vma_category, p)) 2309 return 1; 2310 2311 p->cur_vma_category = vma_category; 2312 2313 return 0; 2314 } 2315 2316 static bool pagemap_scan_push_range(unsigned long categories, 2317 struct pagemap_scan_private *p, 2318 unsigned long addr, unsigned long end) 2319 { 2320 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; 2321 2322 /* 2323 * When there is no output buffer provided at all, the sentinel values 2324 * won't match here. There is no other way for `cur_buf->end` to be 2325 * non-zero other than it being non-empty. 2326 */ 2327 if (addr == cur_buf->end && categories == cur_buf->categories) { 2328 cur_buf->end = end; 2329 return true; 2330 } 2331 2332 if (cur_buf->end) { 2333 if (p->vec_buf_index >= p->vec_buf_len - 1) 2334 return false; 2335 2336 cur_buf = &p->vec_buf[++p->vec_buf_index]; 2337 } 2338 2339 cur_buf->start = addr; 2340 cur_buf->end = end; 2341 cur_buf->categories = categories; 2342 2343 return true; 2344 } 2345 2346 static int pagemap_scan_output(unsigned long categories, 2347 struct pagemap_scan_private *p, 2348 unsigned long addr, unsigned long *end) 2349 { 2350 unsigned long n_pages, total_pages; 2351 int ret = 0; 2352 2353 if (!p->vec_buf) 2354 return 0; 2355 2356 categories &= p->arg.return_mask; 2357 2358 n_pages = (*end - addr) / PAGE_SIZE; 2359 if (check_add_overflow(p->found_pages, n_pages, &total_pages) || 2360 total_pages > p->arg.max_pages) { 2361 size_t n_too_much = total_pages - p->arg.max_pages; 2362 *end -= n_too_much * PAGE_SIZE; 2363 n_pages -= n_too_much; 2364 ret = -ENOSPC; 2365 } 2366 2367 if (!pagemap_scan_push_range(categories, p, addr, *end)) { 2368 *end = addr; 2369 n_pages = 0; 2370 ret = -ENOSPC; 2371 } 2372 2373 p->found_pages += n_pages; 2374 if (ret) 2375 p->arg.walk_end = *end; 2376 2377 return ret; 2378 } 2379 2380 static int pagemap_scan_thp_entry(pmd_t *pmd, unsigned long start, 2381 unsigned long end, struct mm_walk *walk) 2382 { 2383 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2384 struct pagemap_scan_private *p = walk->private; 2385 struct vm_area_struct *vma = walk->vma; 2386 unsigned long categories; 2387 spinlock_t *ptl; 2388 int ret = 0; 2389 2390 ptl = pmd_trans_huge_lock(pmd, vma); 2391 if (!ptl) 2392 return -ENOENT; 2393 2394 categories = p->cur_vma_category | 2395 pagemap_thp_category(p, vma, start, *pmd); 2396 2397 if (!pagemap_scan_is_interesting_page(categories, p)) 2398 goto out_unlock; 2399 2400 ret = pagemap_scan_output(categories, p, start, &end); 2401 if (start == end) 2402 goto out_unlock; 2403 2404 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2405 goto out_unlock; 2406 if (~categories & PAGE_IS_WRITTEN) 2407 goto out_unlock; 2408 2409 /* 2410 * Break huge page into small pages if the WP operation 2411 * needs to be performed on a portion of the huge page. 2412 */ 2413 if (end != start + HPAGE_SIZE) { 2414 spin_unlock(ptl); 2415 split_huge_pmd(vma, pmd, start); 2416 pagemap_scan_backout_range(p, start, end); 2417 /* Report as if there was no THP */ 2418 return -ENOENT; 2419 } 2420 2421 make_uffd_wp_pmd(vma, start, pmd); 2422 flush_tlb_range(vma, start, end); 2423 out_unlock: 2424 spin_unlock(ptl); 2425 return ret; 2426 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 2427 return -ENOENT; 2428 #endif 2429 } 2430 2431 static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, 2432 unsigned long end, struct mm_walk *walk) 2433 { 2434 struct pagemap_scan_private *p = walk->private; 2435 struct vm_area_struct *vma = walk->vma; 2436 unsigned long addr, flush_end = 0; 2437 pte_t *pte, *start_pte; 2438 spinlock_t *ptl; 2439 int ret; 2440 2441 arch_enter_lazy_mmu_mode(); 2442 2443 ret = pagemap_scan_thp_entry(pmd, start, end, walk); 2444 if (ret != -ENOENT) { 2445 arch_leave_lazy_mmu_mode(); 2446 return ret; 2447 } 2448 2449 ret = 0; 2450 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl); 2451 if (!pte) { 2452 arch_leave_lazy_mmu_mode(); 2453 walk->action = ACTION_AGAIN; 2454 return 0; 2455 } 2456 2457 if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { 2458 /* Fast path for performing exclusive WP */ 2459 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { 2460 pte_t ptent = ptep_get(pte); 2461 2462 if ((pte_present(ptent) && pte_uffd_wp(ptent)) || 2463 pte_swp_uffd_wp_any(ptent)) 2464 continue; 2465 make_uffd_wp_pte(vma, addr, pte, ptent); 2466 if (!flush_end) 2467 start = addr; 2468 flush_end = addr + PAGE_SIZE; 2469 } 2470 goto flush_and_return; 2471 } 2472 2473 if (!p->arg.category_anyof_mask && !p->arg.category_inverted && 2474 p->arg.category_mask == PAGE_IS_WRITTEN && 2475 p->arg.return_mask == PAGE_IS_WRITTEN) { 2476 for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { 2477 unsigned long next = addr + PAGE_SIZE; 2478 pte_t ptent = ptep_get(pte); 2479 2480 if ((pte_present(ptent) && pte_uffd_wp(ptent)) || 2481 pte_swp_uffd_wp_any(ptent)) 2482 continue; 2483 ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN, 2484 p, addr, &next); 2485 if (next == addr) 2486 break; 2487 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2488 continue; 2489 make_uffd_wp_pte(vma, addr, pte, ptent); 2490 if (!flush_end) 2491 start = addr; 2492 flush_end = next; 2493 } 2494 goto flush_and_return; 2495 } 2496 2497 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { 2498 pte_t ptent = ptep_get(pte); 2499 unsigned long categories = p->cur_vma_category | 2500 pagemap_page_category(p, vma, addr, ptent); 2501 unsigned long next = addr + PAGE_SIZE; 2502 2503 if (!pagemap_scan_is_interesting_page(categories, p)) 2504 continue; 2505 2506 ret = pagemap_scan_output(categories, p, addr, &next); 2507 if (next == addr) 2508 break; 2509 2510 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2511 continue; 2512 if (~categories & PAGE_IS_WRITTEN) 2513 continue; 2514 2515 make_uffd_wp_pte(vma, addr, pte, ptent); 2516 if (!flush_end) 2517 start = addr; 2518 flush_end = next; 2519 } 2520 2521 flush_and_return: 2522 if (flush_end) 2523 flush_tlb_range(vma, start, addr); 2524 2525 pte_unmap_unlock(start_pte, ptl); 2526 arch_leave_lazy_mmu_mode(); 2527 2528 cond_resched(); 2529 return ret; 2530 } 2531 2532 #ifdef CONFIG_HUGETLB_PAGE 2533 static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, 2534 unsigned long start, unsigned long end, 2535 struct mm_walk *walk) 2536 { 2537 struct pagemap_scan_private *p = walk->private; 2538 struct vm_area_struct *vma = walk->vma; 2539 unsigned long categories; 2540 spinlock_t *ptl; 2541 int ret = 0; 2542 pte_t pte; 2543 2544 if (~p->arg.flags & PM_SCAN_WP_MATCHING) { 2545 /* Go the short route when not write-protecting pages. */ 2546 2547 pte = huge_ptep_get(ptep); 2548 categories = p->cur_vma_category | pagemap_hugetlb_category(pte); 2549 2550 if (!pagemap_scan_is_interesting_page(categories, p)) 2551 return 0; 2552 2553 return pagemap_scan_output(categories, p, start, &end); 2554 } 2555 2556 i_mmap_lock_write(vma->vm_file->f_mapping); 2557 ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep); 2558 2559 pte = huge_ptep_get(ptep); 2560 categories = p->cur_vma_category | pagemap_hugetlb_category(pte); 2561 2562 if (!pagemap_scan_is_interesting_page(categories, p)) 2563 goto out_unlock; 2564 2565 ret = pagemap_scan_output(categories, p, start, &end); 2566 if (start == end) 2567 goto out_unlock; 2568 2569 if (~categories & PAGE_IS_WRITTEN) 2570 goto out_unlock; 2571 2572 if (end != start + HPAGE_SIZE) { 2573 /* Partial HugeTLB page WP isn't possible. */ 2574 pagemap_scan_backout_range(p, start, end); 2575 p->arg.walk_end = start; 2576 ret = 0; 2577 goto out_unlock; 2578 } 2579 2580 make_uffd_wp_huge_pte(vma, start, ptep, pte); 2581 flush_hugetlb_tlb_range(vma, start, end); 2582 2583 out_unlock: 2584 spin_unlock(ptl); 2585 i_mmap_unlock_write(vma->vm_file->f_mapping); 2586 2587 return ret; 2588 } 2589 #else 2590 #define pagemap_scan_hugetlb_entry NULL 2591 #endif 2592 2593 static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, 2594 int depth, struct mm_walk *walk) 2595 { 2596 struct pagemap_scan_private *p = walk->private; 2597 struct vm_area_struct *vma = walk->vma; 2598 int ret, err; 2599 2600 if (!vma || !pagemap_scan_is_interesting_page(p->cur_vma_category, p)) 2601 return 0; 2602 2603 ret = pagemap_scan_output(p->cur_vma_category, p, addr, &end); 2604 if (addr == end) 2605 return ret; 2606 2607 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2608 return ret; 2609 2610 err = uffd_wp_range(vma, addr, end - addr, true); 2611 if (err < 0) 2612 ret = err; 2613 2614 return ret; 2615 } 2616 2617 static const struct mm_walk_ops pagemap_scan_ops = { 2618 .test_walk = pagemap_scan_test_walk, 2619 .pmd_entry = pagemap_scan_pmd_entry, 2620 .pte_hole = pagemap_scan_pte_hole, 2621 .hugetlb_entry = pagemap_scan_hugetlb_entry, 2622 }; 2623 2624 static int pagemap_scan_get_args(struct pm_scan_arg *arg, 2625 unsigned long uarg) 2626 { 2627 if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) 2628 return -EFAULT; 2629 2630 if (arg->size != sizeof(struct pm_scan_arg)) 2631 return -EINVAL; 2632 2633 /* Validate requested features */ 2634 if (arg->flags & ~PM_SCAN_FLAGS) 2635 return -EINVAL; 2636 if ((arg->category_inverted | arg->category_mask | 2637 arg->category_anyof_mask | arg->return_mask) & ~PM_SCAN_CATEGORIES) 2638 return -EINVAL; 2639 2640 arg->start = untagged_addr((unsigned long)arg->start); 2641 arg->end = untagged_addr((unsigned long)arg->end); 2642 arg->vec = untagged_addr((unsigned long)arg->vec); 2643 2644 /* Validate memory pointers */ 2645 if (!IS_ALIGNED(arg->start, PAGE_SIZE)) 2646 return -EINVAL; 2647 if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) 2648 return -EFAULT; 2649 if (!arg->vec && arg->vec_len) 2650 return -EINVAL; 2651 if (arg->vec && !access_ok((void __user *)(long)arg->vec, 2652 arg->vec_len * sizeof(struct page_region))) 2653 return -EFAULT; 2654 2655 /* Fixup default values */ 2656 arg->end = ALIGN(arg->end, PAGE_SIZE); 2657 arg->walk_end = 0; 2658 if (!arg->max_pages) 2659 arg->max_pages = ULONG_MAX; 2660 2661 return 0; 2662 } 2663 2664 static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, 2665 unsigned long uargl) 2666 { 2667 struct pm_scan_arg __user *uarg = (void __user *)uargl; 2668 2669 if (copy_to_user(&uarg->walk_end, &arg->walk_end, sizeof(arg->walk_end))) 2670 return -EFAULT; 2671 2672 return 0; 2673 } 2674 2675 static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) 2676 { 2677 if (!p->arg.vec_len) 2678 return 0; 2679 2680 p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, 2681 p->arg.vec_len); 2682 p->vec_buf = kmalloc_array(p->vec_buf_len, sizeof(*p->vec_buf), 2683 GFP_KERNEL); 2684 if (!p->vec_buf) 2685 return -ENOMEM; 2686 2687 p->vec_buf->start = p->vec_buf->end = 0; 2688 p->vec_out = (struct page_region __user *)(long)p->arg.vec; 2689 2690 return 0; 2691 } 2692 2693 static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) 2694 { 2695 const struct page_region *buf = p->vec_buf; 2696 long n = p->vec_buf_index; 2697 2698 if (!p->vec_buf) 2699 return 0; 2700 2701 if (buf[n].end != buf[n].start) 2702 n++; 2703 2704 if (!n) 2705 return 0; 2706 2707 if (copy_to_user(p->vec_out, buf, n * sizeof(*buf))) 2708 return -EFAULT; 2709 2710 p->arg.vec_len -= n; 2711 p->vec_out += n; 2712 2713 p->vec_buf_index = 0; 2714 p->vec_buf_len = min_t(size_t, p->vec_buf_len, p->arg.vec_len); 2715 p->vec_buf->start = p->vec_buf->end = 0; 2716 2717 return n; 2718 } 2719 2720 static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) 2721 { 2722 struct pagemap_scan_private p = {0}; 2723 unsigned long walk_start; 2724 size_t n_ranges_out = 0; 2725 int ret; 2726 2727 ret = pagemap_scan_get_args(&p.arg, uarg); 2728 if (ret) 2729 return ret; 2730 2731 p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | 2732 p.arg.return_mask; 2733 ret = pagemap_scan_init_bounce_buffer(&p); 2734 if (ret) 2735 return ret; 2736 2737 for (walk_start = p.arg.start; walk_start < p.arg.end; 2738 walk_start = p.arg.walk_end) { 2739 struct mmu_notifier_range range; 2740 long n_out; 2741 2742 if (fatal_signal_pending(current)) { 2743 ret = -EINTR; 2744 break; 2745 } 2746 2747 ret = mmap_read_lock_killable(mm); 2748 if (ret) 2749 break; 2750 2751 /* Protection change for the range is going to happen. */ 2752 if (p.arg.flags & PM_SCAN_WP_MATCHING) { 2753 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0, 2754 mm, walk_start, p.arg.end); 2755 mmu_notifier_invalidate_range_start(&range); 2756 } 2757 2758 ret = walk_page_range(mm, walk_start, p.arg.end, 2759 &pagemap_scan_ops, &p); 2760 2761 if (p.arg.flags & PM_SCAN_WP_MATCHING) 2762 mmu_notifier_invalidate_range_end(&range); 2763 2764 mmap_read_unlock(mm); 2765 2766 n_out = pagemap_scan_flush_buffer(&p); 2767 if (n_out < 0) 2768 ret = n_out; 2769 else 2770 n_ranges_out += n_out; 2771 2772 if (ret != -ENOSPC) 2773 break; 2774 2775 if (p.arg.vec_len == 0 || p.found_pages == p.arg.max_pages) 2776 break; 2777 } 2778 2779 /* ENOSPC signifies early stop (buffer full) from the walk. */ 2780 if (!ret || ret == -ENOSPC) 2781 ret = n_ranges_out; 2782 2783 /* The walk_end isn't set when ret is zero */ 2784 if (!p.arg.walk_end) 2785 p.arg.walk_end = p.arg.end; 2786 if (pagemap_scan_writeback_args(&p.arg, uarg)) 2787 ret = -EFAULT; 2788 2789 kfree(p.vec_buf); 2790 return ret; 2791 } 2792 2793 static long do_pagemap_cmd(struct file *file, unsigned int cmd, 2794 unsigned long arg) 2795 { 2796 struct mm_struct *mm = file->private_data; 2797 2798 switch (cmd) { 2799 case PAGEMAP_SCAN: 2800 return do_pagemap_scan(mm, arg); 2801 2802 default: 2803 return -EINVAL; 2804 } 2805 } 2806 2807 const struct file_operations proc_pagemap_operations = { 2808 .llseek = mem_lseek, /* borrow this */ 2809 .read = pagemap_read, 2810 .open = pagemap_open, 2811 .release = pagemap_release, 2812 .unlocked_ioctl = do_pagemap_cmd, 2813 .compat_ioctl = do_pagemap_cmd, 2814 }; 2815 #endif /* CONFIG_PROC_PAGE_MONITOR */ 2816 2817 #ifdef CONFIG_NUMA 2818 2819 struct numa_maps { 2820 unsigned long pages; 2821 unsigned long anon; 2822 unsigned long active; 2823 unsigned long writeback; 2824 unsigned long mapcount_max; 2825 unsigned long dirty; 2826 unsigned long swapcache; 2827 unsigned long node[MAX_NUMNODES]; 2828 }; 2829 2830 struct numa_maps_private { 2831 struct proc_maps_private proc_maps; 2832 struct numa_maps md; 2833 }; 2834 2835 static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, 2836 unsigned long nr_pages) 2837 { 2838 struct folio *folio = page_folio(page); 2839 int count = folio_precise_page_mapcount(folio, page); 2840 2841 md->pages += nr_pages; 2842 if (pte_dirty || folio_test_dirty(folio)) 2843 md->dirty += nr_pages; 2844 2845 if (folio_test_swapcache(folio)) 2846 md->swapcache += nr_pages; 2847 2848 if (folio_test_active(folio) || folio_test_unevictable(folio)) 2849 md->active += nr_pages; 2850 2851 if (folio_test_writeback(folio)) 2852 md->writeback += nr_pages; 2853 2854 if (folio_test_anon(folio)) 2855 md->anon += nr_pages; 2856 2857 if (count > md->mapcount_max) 2858 md->mapcount_max = count; 2859 2860 md->node[folio_nid(folio)] += nr_pages; 2861 } 2862 2863 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma, 2864 unsigned long addr) 2865 { 2866 struct page *page; 2867 int nid; 2868 2869 if (!pte_present(pte)) 2870 return NULL; 2871 2872 page = vm_normal_page(vma, addr, pte); 2873 if (!page || is_zone_device_page(page)) 2874 return NULL; 2875 2876 if (PageReserved(page)) 2877 return NULL; 2878 2879 nid = page_to_nid(page); 2880 if (!node_isset(nid, node_states[N_MEMORY])) 2881 return NULL; 2882 2883 return page; 2884 } 2885 2886 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2887 static struct page *can_gather_numa_stats_pmd(pmd_t pmd, 2888 struct vm_area_struct *vma, 2889 unsigned long addr) 2890 { 2891 struct page *page; 2892 int nid; 2893 2894 if (!pmd_present(pmd)) 2895 return NULL; 2896 2897 page = vm_normal_page_pmd(vma, addr, pmd); 2898 if (!page) 2899 return NULL; 2900 2901 if (PageReserved(page)) 2902 return NULL; 2903 2904 nid = page_to_nid(page); 2905 if (!node_isset(nid, node_states[N_MEMORY])) 2906 return NULL; 2907 2908 return page; 2909 } 2910 #endif 2911 2912 static int gather_pte_stats(pmd_t *pmd, unsigned long addr, 2913 unsigned long end, struct mm_walk *walk) 2914 { 2915 struct numa_maps *md = walk->private; 2916 struct vm_area_struct *vma = walk->vma; 2917 spinlock_t *ptl; 2918 pte_t *orig_pte; 2919 pte_t *pte; 2920 2921 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2922 ptl = pmd_trans_huge_lock(pmd, vma); 2923 if (ptl) { 2924 struct page *page; 2925 2926 page = can_gather_numa_stats_pmd(*pmd, vma, addr); 2927 if (page) 2928 gather_stats(page, md, pmd_dirty(*pmd), 2929 HPAGE_PMD_SIZE/PAGE_SIZE); 2930 spin_unlock(ptl); 2931 return 0; 2932 } 2933 #endif 2934 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); 2935 if (!pte) { 2936 walk->action = ACTION_AGAIN; 2937 return 0; 2938 } 2939 do { 2940 pte_t ptent = ptep_get(pte); 2941 struct page *page = can_gather_numa_stats(ptent, vma, addr); 2942 if (!page) 2943 continue; 2944 gather_stats(page, md, pte_dirty(ptent), 1); 2945 2946 } while (pte++, addr += PAGE_SIZE, addr != end); 2947 pte_unmap_unlock(orig_pte, ptl); 2948 cond_resched(); 2949 return 0; 2950 } 2951 #ifdef CONFIG_HUGETLB_PAGE 2952 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, 2953 unsigned long addr, unsigned long end, struct mm_walk *walk) 2954 { 2955 pte_t huge_pte = huge_ptep_get(pte); 2956 struct numa_maps *md; 2957 struct page *page; 2958 2959 if (!pte_present(huge_pte)) 2960 return 0; 2961 2962 page = pte_page(huge_pte); 2963 2964 md = walk->private; 2965 gather_stats(page, md, pte_dirty(huge_pte), 1); 2966 return 0; 2967 } 2968 2969 #else 2970 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, 2971 unsigned long addr, unsigned long end, struct mm_walk *walk) 2972 { 2973 return 0; 2974 } 2975 #endif 2976 2977 static const struct mm_walk_ops show_numa_ops = { 2978 .hugetlb_entry = gather_hugetlb_stats, 2979 .pmd_entry = gather_pte_stats, 2980 .walk_lock = PGWALK_RDLOCK, 2981 }; 2982 2983 /* 2984 * Display pages allocated per node and memory policy via /proc. 2985 */ 2986 static int show_numa_map(struct seq_file *m, void *v) 2987 { 2988 struct numa_maps_private *numa_priv = m->private; 2989 struct proc_maps_private *proc_priv = &numa_priv->proc_maps; 2990 struct vm_area_struct *vma = v; 2991 struct numa_maps *md = &numa_priv->md; 2992 struct file *file = vma->vm_file; 2993 struct mm_struct *mm = vma->vm_mm; 2994 char buffer[64]; 2995 struct mempolicy *pol; 2996 pgoff_t ilx; 2997 int nid; 2998 2999 if (!mm) 3000 return 0; 3001 3002 /* Ensure we start with an empty set of numa_maps statistics. */ 3003 memset(md, 0, sizeof(*md)); 3004 3005 pol = __get_vma_policy(vma, vma->vm_start, &ilx); 3006 if (pol) { 3007 mpol_to_str(buffer, sizeof(buffer), pol); 3008 mpol_cond_put(pol); 3009 } else { 3010 mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy); 3011 } 3012 3013 seq_printf(m, "%08lx %s", vma->vm_start, buffer); 3014 3015 if (file) { 3016 seq_puts(m, " file="); 3017 seq_path(m, file_user_path(file), "\n\t= "); 3018 } else if (vma_is_initial_heap(vma)) { 3019 seq_puts(m, " heap"); 3020 } else if (vma_is_initial_stack(vma)) { 3021 seq_puts(m, " stack"); 3022 } 3023 3024 if (is_vm_hugetlb_page(vma)) 3025 seq_puts(m, " huge"); 3026 3027 /* mmap_lock is held by m_start */ 3028 walk_page_vma(vma, &show_numa_ops, md); 3029 3030 if (!md->pages) 3031 goto out; 3032 3033 if (md->anon) 3034 seq_printf(m, " anon=%lu", md->anon); 3035 3036 if (md->dirty) 3037 seq_printf(m, " dirty=%lu", md->dirty); 3038 3039 if (md->pages != md->anon && md->pages != md->dirty) 3040 seq_printf(m, " mapped=%lu", md->pages); 3041 3042 if (md->mapcount_max > 1) 3043 seq_printf(m, " mapmax=%lu", md->mapcount_max); 3044 3045 if (md->swapcache) 3046 seq_printf(m, " swapcache=%lu", md->swapcache); 3047 3048 if (md->active < md->pages && !is_vm_hugetlb_page(vma)) 3049 seq_printf(m, " active=%lu", md->active); 3050 3051 if (md->writeback) 3052 seq_printf(m, " writeback=%lu", md->writeback); 3053 3054 for_each_node_state(nid, N_MEMORY) 3055 if (md->node[nid]) 3056 seq_printf(m, " N%d=%lu", nid, md->node[nid]); 3057 3058 seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10); 3059 out: 3060 seq_putc(m, '\n'); 3061 return 0; 3062 } 3063 3064 static const struct seq_operations proc_pid_numa_maps_op = { 3065 .start = m_start, 3066 .next = m_next, 3067 .stop = m_stop, 3068 .show = show_numa_map, 3069 }; 3070 3071 static int pid_numa_maps_open(struct inode *inode, struct file *file) 3072 { 3073 return proc_maps_open(inode, file, &proc_pid_numa_maps_op, 3074 sizeof(struct numa_maps_private)); 3075 } 3076 3077 const struct file_operations proc_pid_numa_maps_operations = { 3078 .open = pid_numa_maps_open, 3079 .read = seq_read, 3080 .llseek = seq_lseek, 3081 .release = proc_map_release, 3082 }; 3083 3084 #endif /* CONFIG_NUMA */ 3085