1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This is a module to test the HMM (Heterogeneous Memory Management) 4 * mirror and zone device private memory migration APIs of the kernel. 5 * Userspace programs can register with the driver to mirror their own address 6 * space and can use the device to read/write any valid virtual address. 7 */ 8 #include <linux/init.h> 9 #include <linux/fs.h> 10 #include <linux/mm.h> 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/memremap.h> 16 #include <linux/mutex.h> 17 #include <linux/rwsem.h> 18 #include <linux/sched.h> 19 #include <linux/slab.h> 20 #include <linux/highmem.h> 21 #include <linux/delay.h> 22 #include <linux/pagemap.h> 23 #include <linux/hmm.h> 24 #include <linux/vmalloc.h> 25 #include <linux/swap.h> 26 #include <linux/swapops.h> 27 #include <linux/sched/mm.h> 28 #include <linux/platform_device.h> 29 #include <linux/rmap.h> 30 #include <linux/mmu_notifier.h> 31 #include <linux/migrate.h> 32 33 #include "test_hmm_uapi.h" 34 35 #define DMIRROR_NDEVICES 4 36 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000 37 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) 38 #define DEVMEM_CHUNKS_RESERVE 16 39 40 /* 41 * For device_private pages, dpage is just a dummy struct page 42 * representing a piece of device memory. dmirror_devmem_alloc_page 43 * allocates a real system memory page as backing storage to fake a 44 * real device. zone_device_data points to that backing page. But 45 * for device_coherent memory, the struct page represents real 46 * physical CPU-accessible memory that we can use directly. 47 */ 48 #define BACKING_PAGE(page) (is_device_private_page((page)) ? \ 49 (page)->zone_device_data : (page)) 50 51 static unsigned long spm_addr_dev0; 52 module_param(spm_addr_dev0, long, 0644); 53 MODULE_PARM_DESC(spm_addr_dev0, 54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 55 56 static unsigned long spm_addr_dev1; 57 module_param(spm_addr_dev1, long, 0644); 58 MODULE_PARM_DESC(spm_addr_dev1, 59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 60 61 static const struct dev_pagemap_ops dmirror_devmem_ops; 62 static const struct mmu_interval_notifier_ops dmirror_min_ops; 63 static dev_t dmirror_dev; 64 65 struct dmirror_device; 66 67 struct dmirror_bounce { 68 void *ptr; 69 unsigned long size; 70 unsigned long addr; 71 unsigned long cpages; 72 }; 73 74 #define DPT_XA_TAG_ATOMIC 1UL 75 #define DPT_XA_TAG_WRITE 3UL 76 77 /* 78 * Data structure to track address ranges and register for mmu interval 79 * notifier updates. 80 */ 81 struct dmirror_interval { 82 struct mmu_interval_notifier notifier; 83 struct dmirror *dmirror; 84 }; 85 86 /* 87 * Data attached to the open device file. 88 * Note that it might be shared after a fork(). 89 */ 90 struct dmirror { 91 struct dmirror_device *mdevice; 92 struct xarray pt; 93 struct mmu_interval_notifier notifier; 94 struct mutex mutex; 95 }; 96 97 /* 98 * ZONE_DEVICE pages for migration and simulating device memory. 99 */ 100 struct dmirror_chunk { 101 struct dev_pagemap pagemap; 102 struct dmirror_device *mdevice; 103 bool remove; 104 }; 105 106 /* 107 * Per device data. 108 */ 109 struct dmirror_device { 110 struct cdev cdevice; 111 unsigned int zone_device_type; 112 struct device device; 113 114 unsigned int devmem_capacity; 115 unsigned int devmem_count; 116 struct dmirror_chunk **devmem_chunks; 117 struct mutex devmem_lock; /* protects the above */ 118 119 unsigned long calloc; 120 unsigned long cfree; 121 struct page *free_pages; 122 spinlock_t lock; /* protects the above */ 123 }; 124 125 static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; 126 127 static int dmirror_bounce_init(struct dmirror_bounce *bounce, 128 unsigned long addr, 129 unsigned long size) 130 { 131 bounce->addr = addr; 132 bounce->size = size; 133 bounce->cpages = 0; 134 bounce->ptr = vmalloc(size); 135 if (!bounce->ptr) 136 return -ENOMEM; 137 return 0; 138 } 139 140 static bool dmirror_is_private_zone(struct dmirror_device *mdevice) 141 { 142 return (mdevice->zone_device_type == 143 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? true : false; 144 } 145 146 static enum migrate_vma_direction 147 dmirror_select_device(struct dmirror *dmirror) 148 { 149 return (dmirror->mdevice->zone_device_type == 150 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? 151 MIGRATE_VMA_SELECT_DEVICE_PRIVATE : 152 MIGRATE_VMA_SELECT_DEVICE_COHERENT; 153 } 154 155 static void dmirror_bounce_fini(struct dmirror_bounce *bounce) 156 { 157 vfree(bounce->ptr); 158 } 159 160 static int dmirror_fops_open(struct inode *inode, struct file *filp) 161 { 162 struct cdev *cdev = inode->i_cdev; 163 struct dmirror *dmirror; 164 int ret; 165 166 /* Mirror this process address space */ 167 dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL); 168 if (dmirror == NULL) 169 return -ENOMEM; 170 171 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); 172 mutex_init(&dmirror->mutex); 173 xa_init(&dmirror->pt); 174 175 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 176 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); 177 if (ret) { 178 kfree(dmirror); 179 return ret; 180 } 181 182 filp->private_data = dmirror; 183 return 0; 184 } 185 186 static int dmirror_fops_release(struct inode *inode, struct file *filp) 187 { 188 struct dmirror *dmirror = filp->private_data; 189 190 mmu_interval_notifier_remove(&dmirror->notifier); 191 xa_destroy(&dmirror->pt); 192 kfree(dmirror); 193 return 0; 194 } 195 196 static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page) 197 { 198 return container_of(page->pgmap, struct dmirror_chunk, pagemap); 199 } 200 201 static struct dmirror_device *dmirror_page_to_device(struct page *page) 202 203 { 204 return dmirror_page_to_chunk(page)->mdevice; 205 } 206 207 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) 208 { 209 unsigned long *pfns = range->hmm_pfns; 210 unsigned long pfn; 211 212 for (pfn = (range->start >> PAGE_SHIFT); 213 pfn < (range->end >> PAGE_SHIFT); 214 pfn++, pfns++) { 215 struct page *page; 216 void *entry; 217 218 /* 219 * Since we asked for hmm_range_fault() to populate pages, 220 * it shouldn't return an error entry on success. 221 */ 222 WARN_ON(*pfns & HMM_PFN_ERROR); 223 WARN_ON(!(*pfns & HMM_PFN_VALID)); 224 225 page = hmm_pfn_to_page(*pfns); 226 WARN_ON(!page); 227 228 entry = page; 229 if (*pfns & HMM_PFN_WRITE) 230 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 231 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) 232 return -EFAULT; 233 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 234 if (xa_is_err(entry)) 235 return xa_err(entry); 236 } 237 238 return 0; 239 } 240 241 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, 242 unsigned long end) 243 { 244 unsigned long pfn; 245 void *entry; 246 247 /* 248 * The XArray doesn't hold references to pages since it relies on 249 * the mmu notifier to clear page pointers when they become stale. 250 * Therefore, it is OK to just clear the entry. 251 */ 252 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, 253 end >> PAGE_SHIFT) 254 xa_erase(&dmirror->pt, pfn); 255 } 256 257 static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, 258 const struct mmu_notifier_range *range, 259 unsigned long cur_seq) 260 { 261 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); 262 263 /* 264 * Ignore invalidation callbacks for device private pages since 265 * the invalidation is handled as part of the migration process. 266 */ 267 if (range->event == MMU_NOTIFY_MIGRATE && 268 range->owner == dmirror->mdevice) 269 return true; 270 271 if (mmu_notifier_range_blockable(range)) 272 mutex_lock(&dmirror->mutex); 273 else if (!mutex_trylock(&dmirror->mutex)) 274 return false; 275 276 mmu_interval_set_seq(mni, cur_seq); 277 dmirror_do_update(dmirror, range->start, range->end); 278 279 mutex_unlock(&dmirror->mutex); 280 return true; 281 } 282 283 static const struct mmu_interval_notifier_ops dmirror_min_ops = { 284 .invalidate = dmirror_interval_invalidate, 285 }; 286 287 static int dmirror_range_fault(struct dmirror *dmirror, 288 struct hmm_range *range) 289 { 290 struct mm_struct *mm = dmirror->notifier.mm; 291 unsigned long timeout = 292 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 293 int ret; 294 295 while (true) { 296 if (time_after(jiffies, timeout)) { 297 ret = -EBUSY; 298 goto out; 299 } 300 301 range->notifier_seq = mmu_interval_read_begin(range->notifier); 302 mmap_read_lock(mm); 303 ret = hmm_range_fault(range); 304 mmap_read_unlock(mm); 305 if (ret) { 306 if (ret == -EBUSY) 307 continue; 308 goto out; 309 } 310 311 mutex_lock(&dmirror->mutex); 312 if (mmu_interval_read_retry(range->notifier, 313 range->notifier_seq)) { 314 mutex_unlock(&dmirror->mutex); 315 continue; 316 } 317 break; 318 } 319 320 ret = dmirror_do_fault(dmirror, range); 321 322 mutex_unlock(&dmirror->mutex); 323 out: 324 return ret; 325 } 326 327 static int dmirror_fault(struct dmirror *dmirror, unsigned long start, 328 unsigned long end, bool write) 329 { 330 struct mm_struct *mm = dmirror->notifier.mm; 331 unsigned long addr; 332 unsigned long pfns[64]; 333 struct hmm_range range = { 334 .notifier = &dmirror->notifier, 335 .hmm_pfns = pfns, 336 .pfn_flags_mask = 0, 337 .default_flags = 338 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 339 .dev_private_owner = dmirror->mdevice, 340 }; 341 int ret = 0; 342 343 /* Since the mm is for the mirrored process, get a reference first. */ 344 if (!mmget_not_zero(mm)) 345 return 0; 346 347 for (addr = start; addr < end; addr = range.end) { 348 range.start = addr; 349 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 350 351 ret = dmirror_range_fault(dmirror, &range); 352 if (ret) 353 break; 354 } 355 356 mmput(mm); 357 return ret; 358 } 359 360 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, 361 unsigned long end, struct dmirror_bounce *bounce) 362 { 363 unsigned long pfn; 364 void *ptr; 365 366 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 367 368 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 369 void *entry; 370 struct page *page; 371 372 entry = xa_load(&dmirror->pt, pfn); 373 page = xa_untag_pointer(entry); 374 if (!page) 375 return -ENOENT; 376 377 memcpy_from_page(ptr, page, 0, PAGE_SIZE); 378 379 ptr += PAGE_SIZE; 380 bounce->cpages++; 381 } 382 383 return 0; 384 } 385 386 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 387 { 388 struct dmirror_bounce bounce; 389 unsigned long start, end; 390 unsigned long size = cmd->npages << PAGE_SHIFT; 391 int ret; 392 393 start = cmd->addr; 394 end = start + size; 395 if (end < start) 396 return -EINVAL; 397 398 ret = dmirror_bounce_init(&bounce, start, size); 399 if (ret) 400 return ret; 401 402 while (1) { 403 mutex_lock(&dmirror->mutex); 404 ret = dmirror_do_read(dmirror, start, end, &bounce); 405 mutex_unlock(&dmirror->mutex); 406 if (ret != -ENOENT) 407 break; 408 409 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 410 ret = dmirror_fault(dmirror, start, end, false); 411 if (ret) 412 break; 413 cmd->faults++; 414 } 415 416 if (ret == 0) { 417 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 418 bounce.size)) 419 ret = -EFAULT; 420 } 421 cmd->cpages = bounce.cpages; 422 dmirror_bounce_fini(&bounce); 423 return ret; 424 } 425 426 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, 427 unsigned long end, struct dmirror_bounce *bounce) 428 { 429 unsigned long pfn; 430 void *ptr; 431 432 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 433 434 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 435 void *entry; 436 struct page *page; 437 438 entry = xa_load(&dmirror->pt, pfn); 439 page = xa_untag_pointer(entry); 440 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) 441 return -ENOENT; 442 443 memcpy_to_page(page, 0, ptr, PAGE_SIZE); 444 445 ptr += PAGE_SIZE; 446 bounce->cpages++; 447 } 448 449 return 0; 450 } 451 452 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 453 { 454 struct dmirror_bounce bounce; 455 unsigned long start, end; 456 unsigned long size = cmd->npages << PAGE_SHIFT; 457 int ret; 458 459 start = cmd->addr; 460 end = start + size; 461 if (end < start) 462 return -EINVAL; 463 464 ret = dmirror_bounce_init(&bounce, start, size); 465 if (ret) 466 return ret; 467 if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), 468 bounce.size)) { 469 ret = -EFAULT; 470 goto fini; 471 } 472 473 while (1) { 474 mutex_lock(&dmirror->mutex); 475 ret = dmirror_do_write(dmirror, start, end, &bounce); 476 mutex_unlock(&dmirror->mutex); 477 if (ret != -ENOENT) 478 break; 479 480 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 481 ret = dmirror_fault(dmirror, start, end, true); 482 if (ret) 483 break; 484 cmd->faults++; 485 } 486 487 fini: 488 cmd->cpages = bounce.cpages; 489 dmirror_bounce_fini(&bounce); 490 return ret; 491 } 492 493 static int dmirror_allocate_chunk(struct dmirror_device *mdevice, 494 struct page **ppage) 495 { 496 struct dmirror_chunk *devmem; 497 struct resource *res = NULL; 498 unsigned long pfn; 499 unsigned long pfn_first; 500 unsigned long pfn_last; 501 void *ptr; 502 int ret = -ENOMEM; 503 504 devmem = kzalloc(sizeof(*devmem), GFP_KERNEL); 505 if (!devmem) 506 return ret; 507 508 switch (mdevice->zone_device_type) { 509 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE: 510 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, 511 "hmm_dmirror"); 512 if (IS_ERR_OR_NULL(res)) 513 goto err_devmem; 514 devmem->pagemap.range.start = res->start; 515 devmem->pagemap.range.end = res->end; 516 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 517 break; 518 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: 519 devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ? 520 spm_addr_dev0 : 521 spm_addr_dev1; 522 devmem->pagemap.range.end = devmem->pagemap.range.start + 523 DEVMEM_CHUNK_SIZE - 1; 524 devmem->pagemap.type = MEMORY_DEVICE_COHERENT; 525 break; 526 default: 527 ret = -EINVAL; 528 goto err_devmem; 529 } 530 531 devmem->pagemap.nr_range = 1; 532 devmem->pagemap.ops = &dmirror_devmem_ops; 533 devmem->pagemap.owner = mdevice; 534 535 mutex_lock(&mdevice->devmem_lock); 536 537 if (mdevice->devmem_count == mdevice->devmem_capacity) { 538 struct dmirror_chunk **new_chunks; 539 unsigned int new_capacity; 540 541 new_capacity = mdevice->devmem_capacity + 542 DEVMEM_CHUNKS_RESERVE; 543 new_chunks = krealloc(mdevice->devmem_chunks, 544 sizeof(new_chunks[0]) * new_capacity, 545 GFP_KERNEL); 546 if (!new_chunks) 547 goto err_release; 548 mdevice->devmem_capacity = new_capacity; 549 mdevice->devmem_chunks = new_chunks; 550 } 551 ptr = memremap_pages(&devmem->pagemap, numa_node_id()); 552 if (IS_ERR_OR_NULL(ptr)) { 553 if (ptr) 554 ret = PTR_ERR(ptr); 555 else 556 ret = -EFAULT; 557 goto err_release; 558 } 559 560 devmem->mdevice = mdevice; 561 pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; 562 pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); 563 mdevice->devmem_chunks[mdevice->devmem_count++] = devmem; 564 565 mutex_unlock(&mdevice->devmem_lock); 566 567 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", 568 DEVMEM_CHUNK_SIZE / (1024 * 1024), 569 mdevice->devmem_count, 570 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), 571 pfn_first, pfn_last); 572 573 spin_lock(&mdevice->lock); 574 for (pfn = pfn_first; pfn < pfn_last; pfn++) { 575 struct page *page = pfn_to_page(pfn); 576 577 page->zone_device_data = mdevice->free_pages; 578 mdevice->free_pages = page; 579 } 580 if (ppage) { 581 *ppage = mdevice->free_pages; 582 mdevice->free_pages = (*ppage)->zone_device_data; 583 mdevice->calloc++; 584 } 585 spin_unlock(&mdevice->lock); 586 587 return 0; 588 589 err_release: 590 mutex_unlock(&mdevice->devmem_lock); 591 if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 592 release_mem_region(devmem->pagemap.range.start, 593 range_len(&devmem->pagemap.range)); 594 err_devmem: 595 kfree(devmem); 596 597 return ret; 598 } 599 600 static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) 601 { 602 struct page *dpage = NULL; 603 struct page *rpage = NULL; 604 605 /* 606 * For ZONE_DEVICE private type, this is a fake device so we allocate 607 * real system memory to store our device memory. 608 * For ZONE_DEVICE coherent type we use the actual dpage to store the 609 * data and ignore rpage. 610 */ 611 if (dmirror_is_private_zone(mdevice)) { 612 rpage = alloc_page(GFP_HIGHUSER); 613 if (!rpage) 614 return NULL; 615 } 616 spin_lock(&mdevice->lock); 617 618 if (mdevice->free_pages) { 619 dpage = mdevice->free_pages; 620 mdevice->free_pages = dpage->zone_device_data; 621 mdevice->calloc++; 622 spin_unlock(&mdevice->lock); 623 } else { 624 spin_unlock(&mdevice->lock); 625 if (dmirror_allocate_chunk(mdevice, &dpage)) 626 goto error; 627 } 628 629 zone_device_page_init(dpage); 630 dpage->zone_device_data = rpage; 631 return dpage; 632 633 error: 634 if (rpage) 635 __free_page(rpage); 636 return NULL; 637 } 638 639 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, 640 struct dmirror *dmirror) 641 { 642 struct dmirror_device *mdevice = dmirror->mdevice; 643 const unsigned long *src = args->src; 644 unsigned long *dst = args->dst; 645 unsigned long addr; 646 647 for (addr = args->start; addr < args->end; addr += PAGE_SIZE, 648 src++, dst++) { 649 struct page *spage; 650 struct page *dpage; 651 struct page *rpage; 652 653 if (!(*src & MIGRATE_PFN_MIGRATE)) 654 continue; 655 656 /* 657 * Note that spage might be NULL which is OK since it is an 658 * unallocated pte_none() or read-only zero page. 659 */ 660 spage = migrate_pfn_to_page(*src); 661 if (WARN(spage && is_zone_device_page(spage), 662 "page already in device spage pfn: 0x%lx\n", 663 page_to_pfn(spage))) 664 continue; 665 666 dpage = dmirror_devmem_alloc_page(mdevice); 667 if (!dpage) 668 continue; 669 670 rpage = BACKING_PAGE(dpage); 671 if (spage) 672 copy_highpage(rpage, spage); 673 else 674 clear_highpage(rpage); 675 676 /* 677 * Normally, a device would use the page->zone_device_data to 678 * point to the mirror but here we use it to hold the page for 679 * the simulated device memory and that page holds the pointer 680 * to the mirror. 681 */ 682 rpage->zone_device_data = dmirror; 683 684 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", 685 page_to_pfn(spage), page_to_pfn(dpage)); 686 *dst = migrate_pfn(page_to_pfn(dpage)); 687 if ((*src & MIGRATE_PFN_WRITE) || 688 (!spage && args->vma->vm_flags & VM_WRITE)) 689 *dst |= MIGRATE_PFN_WRITE; 690 } 691 } 692 693 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, 694 unsigned long end) 695 { 696 unsigned long pfn; 697 698 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 699 void *entry; 700 701 entry = xa_load(&dmirror->pt, pfn); 702 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) 703 return -EPERM; 704 } 705 706 return 0; 707 } 708 709 static int dmirror_atomic_map(unsigned long addr, struct page *page, 710 struct dmirror *dmirror) 711 { 712 void *entry; 713 714 /* Map the migrated pages into the device's page tables. */ 715 mutex_lock(&dmirror->mutex); 716 717 entry = xa_tag_pointer(page, DPT_XA_TAG_ATOMIC); 718 entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC); 719 if (xa_is_err(entry)) { 720 mutex_unlock(&dmirror->mutex); 721 return xa_err(entry); 722 } 723 724 mutex_unlock(&dmirror->mutex); 725 return 0; 726 } 727 728 static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, 729 struct dmirror *dmirror) 730 { 731 unsigned long start = args->start; 732 unsigned long end = args->end; 733 const unsigned long *src = args->src; 734 const unsigned long *dst = args->dst; 735 unsigned long pfn; 736 737 /* Map the migrated pages into the device's page tables. */ 738 mutex_lock(&dmirror->mutex); 739 740 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, 741 src++, dst++) { 742 struct page *dpage; 743 void *entry; 744 745 if (!(*src & MIGRATE_PFN_MIGRATE)) 746 continue; 747 748 dpage = migrate_pfn_to_page(*dst); 749 if (!dpage) 750 continue; 751 752 entry = BACKING_PAGE(dpage); 753 if (*dst & MIGRATE_PFN_WRITE) 754 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 755 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 756 if (xa_is_err(entry)) { 757 mutex_unlock(&dmirror->mutex); 758 return xa_err(entry); 759 } 760 } 761 762 mutex_unlock(&dmirror->mutex); 763 return 0; 764 } 765 766 static int dmirror_exclusive(struct dmirror *dmirror, 767 struct hmm_dmirror_cmd *cmd) 768 { 769 unsigned long start, end, addr; 770 unsigned long size = cmd->npages << PAGE_SHIFT; 771 struct mm_struct *mm = dmirror->notifier.mm; 772 struct dmirror_bounce bounce; 773 int ret = 0; 774 775 start = cmd->addr; 776 end = start + size; 777 if (end < start) 778 return -EINVAL; 779 780 /* Since the mm is for the mirrored process, get a reference first. */ 781 if (!mmget_not_zero(mm)) 782 return -EINVAL; 783 784 mmap_read_lock(mm); 785 for (addr = start; !ret && addr < end; addr += PAGE_SIZE) { 786 struct folio *folio; 787 struct page *page; 788 789 page = make_device_exclusive(mm, addr, NULL, &folio); 790 if (IS_ERR(page)) { 791 ret = PTR_ERR(page); 792 break; 793 } 794 795 ret = dmirror_atomic_map(addr, page, dmirror); 796 folio_unlock(folio); 797 folio_put(folio); 798 } 799 mmap_read_unlock(mm); 800 mmput(mm); 801 802 if (ret) 803 return ret; 804 805 /* Return the migrated data for verification. */ 806 ret = dmirror_bounce_init(&bounce, start, size); 807 if (ret) 808 return ret; 809 mutex_lock(&dmirror->mutex); 810 ret = dmirror_do_read(dmirror, start, end, &bounce); 811 mutex_unlock(&dmirror->mutex); 812 if (ret == 0) { 813 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 814 bounce.size)) 815 ret = -EFAULT; 816 } 817 818 cmd->cpages = bounce.cpages; 819 dmirror_bounce_fini(&bounce); 820 return ret; 821 } 822 823 static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, 824 struct dmirror *dmirror) 825 { 826 const unsigned long *src = args->src; 827 unsigned long *dst = args->dst; 828 unsigned long start = args->start; 829 unsigned long end = args->end; 830 unsigned long addr; 831 832 for (addr = start; addr < end; addr += PAGE_SIZE, 833 src++, dst++) { 834 struct page *dpage, *spage; 835 836 spage = migrate_pfn_to_page(*src); 837 if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) 838 continue; 839 840 if (WARN_ON(!is_device_private_page(spage) && 841 !is_device_coherent_page(spage))) 842 continue; 843 spage = BACKING_PAGE(spage); 844 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 845 if (!dpage) 846 continue; 847 pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n", 848 page_to_pfn(spage), page_to_pfn(dpage)); 849 850 lock_page(dpage); 851 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); 852 copy_highpage(dpage, spage); 853 *dst = migrate_pfn(page_to_pfn(dpage)); 854 if (*src & MIGRATE_PFN_WRITE) 855 *dst |= MIGRATE_PFN_WRITE; 856 } 857 return 0; 858 } 859 860 static unsigned long 861 dmirror_successful_migrated_pages(struct migrate_vma *migrate) 862 { 863 unsigned long cpages = 0; 864 unsigned long i; 865 866 for (i = 0; i < migrate->npages; i++) { 867 if (migrate->src[i] & MIGRATE_PFN_VALID && 868 migrate->src[i] & MIGRATE_PFN_MIGRATE) 869 cpages++; 870 } 871 return cpages; 872 } 873 874 static int dmirror_migrate_to_system(struct dmirror *dmirror, 875 struct hmm_dmirror_cmd *cmd) 876 { 877 unsigned long start, end, addr; 878 unsigned long size = cmd->npages << PAGE_SHIFT; 879 struct mm_struct *mm = dmirror->notifier.mm; 880 struct vm_area_struct *vma; 881 unsigned long src_pfns[64] = { 0 }; 882 unsigned long dst_pfns[64] = { 0 }; 883 struct migrate_vma args = { 0 }; 884 unsigned long next; 885 int ret; 886 887 start = cmd->addr; 888 end = start + size; 889 if (end < start) 890 return -EINVAL; 891 892 /* Since the mm is for the mirrored process, get a reference first. */ 893 if (!mmget_not_zero(mm)) 894 return -EINVAL; 895 896 cmd->cpages = 0; 897 mmap_read_lock(mm); 898 for (addr = start; addr < end; addr = next) { 899 vma = vma_lookup(mm, addr); 900 if (!vma || !(vma->vm_flags & VM_READ)) { 901 ret = -EINVAL; 902 goto out; 903 } 904 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); 905 if (next > vma->vm_end) 906 next = vma->vm_end; 907 908 args.vma = vma; 909 args.src = src_pfns; 910 args.dst = dst_pfns; 911 args.start = addr; 912 args.end = next; 913 args.pgmap_owner = dmirror->mdevice; 914 args.flags = dmirror_select_device(dmirror); 915 916 ret = migrate_vma_setup(&args); 917 if (ret) 918 goto out; 919 920 pr_debug("Migrating from device mem to sys mem\n"); 921 dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 922 923 migrate_vma_pages(&args); 924 cmd->cpages += dmirror_successful_migrated_pages(&args); 925 migrate_vma_finalize(&args); 926 } 927 out: 928 mmap_read_unlock(mm); 929 mmput(mm); 930 931 return ret; 932 } 933 934 static int dmirror_migrate_to_device(struct dmirror *dmirror, 935 struct hmm_dmirror_cmd *cmd) 936 { 937 unsigned long start, end, addr; 938 unsigned long size = cmd->npages << PAGE_SHIFT; 939 struct mm_struct *mm = dmirror->notifier.mm; 940 struct vm_area_struct *vma; 941 unsigned long src_pfns[64] = { 0 }; 942 unsigned long dst_pfns[64] = { 0 }; 943 struct dmirror_bounce bounce; 944 struct migrate_vma args = { 0 }; 945 unsigned long next; 946 int ret; 947 948 start = cmd->addr; 949 end = start + size; 950 if (end < start) 951 return -EINVAL; 952 953 /* Since the mm is for the mirrored process, get a reference first. */ 954 if (!mmget_not_zero(mm)) 955 return -EINVAL; 956 957 mmap_read_lock(mm); 958 for (addr = start; addr < end; addr = next) { 959 vma = vma_lookup(mm, addr); 960 if (!vma || !(vma->vm_flags & VM_READ)) { 961 ret = -EINVAL; 962 goto out; 963 } 964 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); 965 if (next > vma->vm_end) 966 next = vma->vm_end; 967 968 args.vma = vma; 969 args.src = src_pfns; 970 args.dst = dst_pfns; 971 args.start = addr; 972 args.end = next; 973 args.pgmap_owner = dmirror->mdevice; 974 args.flags = MIGRATE_VMA_SELECT_SYSTEM; 975 ret = migrate_vma_setup(&args); 976 if (ret) 977 goto out; 978 979 pr_debug("Migrating from sys mem to device mem\n"); 980 dmirror_migrate_alloc_and_copy(&args, dmirror); 981 migrate_vma_pages(&args); 982 dmirror_migrate_finalize_and_map(&args, dmirror); 983 migrate_vma_finalize(&args); 984 } 985 mmap_read_unlock(mm); 986 mmput(mm); 987 988 /* 989 * Return the migrated data for verification. 990 * Only for pages in device zone 991 */ 992 ret = dmirror_bounce_init(&bounce, start, size); 993 if (ret) 994 return ret; 995 mutex_lock(&dmirror->mutex); 996 ret = dmirror_do_read(dmirror, start, end, &bounce); 997 mutex_unlock(&dmirror->mutex); 998 if (ret == 0) { 999 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1000 bounce.size)) 1001 ret = -EFAULT; 1002 } 1003 cmd->cpages = bounce.cpages; 1004 dmirror_bounce_fini(&bounce); 1005 return ret; 1006 1007 out: 1008 mmap_read_unlock(mm); 1009 mmput(mm); 1010 return ret; 1011 } 1012 1013 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, 1014 unsigned char *perm, unsigned long entry) 1015 { 1016 struct page *page; 1017 1018 if (entry & HMM_PFN_ERROR) { 1019 *perm = HMM_DMIRROR_PROT_ERROR; 1020 return; 1021 } 1022 if (!(entry & HMM_PFN_VALID)) { 1023 *perm = HMM_DMIRROR_PROT_NONE; 1024 return; 1025 } 1026 1027 page = hmm_pfn_to_page(entry); 1028 if (is_device_private_page(page)) { 1029 /* Is the page migrated to this device or some other? */ 1030 if (dmirror->mdevice == dmirror_page_to_device(page)) 1031 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; 1032 else 1033 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; 1034 } else if (is_device_coherent_page(page)) { 1035 /* Is the page migrated to this device or some other? */ 1036 if (dmirror->mdevice == dmirror_page_to_device(page)) 1037 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL; 1038 else 1039 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE; 1040 } else if (is_zero_pfn(page_to_pfn(page))) 1041 *perm = HMM_DMIRROR_PROT_ZERO; 1042 else 1043 *perm = HMM_DMIRROR_PROT_NONE; 1044 if (entry & HMM_PFN_WRITE) 1045 *perm |= HMM_DMIRROR_PROT_WRITE; 1046 else 1047 *perm |= HMM_DMIRROR_PROT_READ; 1048 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) 1049 *perm |= HMM_DMIRROR_PROT_PMD; 1050 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) 1051 *perm |= HMM_DMIRROR_PROT_PUD; 1052 } 1053 1054 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, 1055 const struct mmu_notifier_range *range, 1056 unsigned long cur_seq) 1057 { 1058 struct dmirror_interval *dmi = 1059 container_of(mni, struct dmirror_interval, notifier); 1060 struct dmirror *dmirror = dmi->dmirror; 1061 1062 if (mmu_notifier_range_blockable(range)) 1063 mutex_lock(&dmirror->mutex); 1064 else if (!mutex_trylock(&dmirror->mutex)) 1065 return false; 1066 1067 /* 1068 * Snapshots only need to set the sequence number since any 1069 * invalidation in the interval invalidates the whole snapshot. 1070 */ 1071 mmu_interval_set_seq(mni, cur_seq); 1072 1073 mutex_unlock(&dmirror->mutex); 1074 return true; 1075 } 1076 1077 static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { 1078 .invalidate = dmirror_snapshot_invalidate, 1079 }; 1080 1081 static int dmirror_range_snapshot(struct dmirror *dmirror, 1082 struct hmm_range *range, 1083 unsigned char *perm) 1084 { 1085 struct mm_struct *mm = dmirror->notifier.mm; 1086 struct dmirror_interval notifier; 1087 unsigned long timeout = 1088 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1089 unsigned long i; 1090 unsigned long n; 1091 int ret = 0; 1092 1093 notifier.dmirror = dmirror; 1094 range->notifier = ¬ifier.notifier; 1095 1096 ret = mmu_interval_notifier_insert(range->notifier, mm, 1097 range->start, range->end - range->start, 1098 &dmirror_mrn_ops); 1099 if (ret) 1100 return ret; 1101 1102 while (true) { 1103 if (time_after(jiffies, timeout)) { 1104 ret = -EBUSY; 1105 goto out; 1106 } 1107 1108 range->notifier_seq = mmu_interval_read_begin(range->notifier); 1109 1110 mmap_read_lock(mm); 1111 ret = hmm_range_fault(range); 1112 mmap_read_unlock(mm); 1113 if (ret) { 1114 if (ret == -EBUSY) 1115 continue; 1116 goto out; 1117 } 1118 1119 mutex_lock(&dmirror->mutex); 1120 if (mmu_interval_read_retry(range->notifier, 1121 range->notifier_seq)) { 1122 mutex_unlock(&dmirror->mutex); 1123 continue; 1124 } 1125 break; 1126 } 1127 1128 n = (range->end - range->start) >> PAGE_SHIFT; 1129 for (i = 0; i < n; i++) 1130 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); 1131 1132 mutex_unlock(&dmirror->mutex); 1133 out: 1134 mmu_interval_notifier_remove(range->notifier); 1135 return ret; 1136 } 1137 1138 static int dmirror_snapshot(struct dmirror *dmirror, 1139 struct hmm_dmirror_cmd *cmd) 1140 { 1141 struct mm_struct *mm = dmirror->notifier.mm; 1142 unsigned long start, end; 1143 unsigned long size = cmd->npages << PAGE_SHIFT; 1144 unsigned long addr; 1145 unsigned long next; 1146 unsigned long pfns[64]; 1147 unsigned char perm[64]; 1148 char __user *uptr; 1149 struct hmm_range range = { 1150 .hmm_pfns = pfns, 1151 .dev_private_owner = dmirror->mdevice, 1152 }; 1153 int ret = 0; 1154 1155 start = cmd->addr; 1156 end = start + size; 1157 if (end < start) 1158 return -EINVAL; 1159 1160 /* Since the mm is for the mirrored process, get a reference first. */ 1161 if (!mmget_not_zero(mm)) 1162 return -EINVAL; 1163 1164 /* 1165 * Register a temporary notifier to detect invalidations even if it 1166 * overlaps with other mmu_interval_notifiers. 1167 */ 1168 uptr = u64_to_user_ptr(cmd->ptr); 1169 for (addr = start; addr < end; addr = next) { 1170 unsigned long n; 1171 1172 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 1173 range.start = addr; 1174 range.end = next; 1175 1176 ret = dmirror_range_snapshot(dmirror, &range, perm); 1177 if (ret) 1178 break; 1179 1180 n = (range.end - range.start) >> PAGE_SHIFT; 1181 if (copy_to_user(uptr, perm, n)) { 1182 ret = -EFAULT; 1183 break; 1184 } 1185 1186 cmd->cpages += n; 1187 uptr += n; 1188 } 1189 mmput(mm); 1190 1191 return ret; 1192 } 1193 1194 static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) 1195 { 1196 unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; 1197 unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; 1198 unsigned long npages = end_pfn - start_pfn + 1; 1199 unsigned long i; 1200 unsigned long *src_pfns; 1201 unsigned long *dst_pfns; 1202 1203 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 1204 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 1205 1206 migrate_device_range(src_pfns, start_pfn, npages); 1207 for (i = 0; i < npages; i++) { 1208 struct page *dpage, *spage; 1209 1210 spage = migrate_pfn_to_page(src_pfns[i]); 1211 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 1212 continue; 1213 1214 if (WARN_ON(!is_device_private_page(spage) && 1215 !is_device_coherent_page(spage))) 1216 continue; 1217 spage = BACKING_PAGE(spage); 1218 dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); 1219 lock_page(dpage); 1220 copy_highpage(dpage, spage); 1221 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); 1222 if (src_pfns[i] & MIGRATE_PFN_WRITE) 1223 dst_pfns[i] |= MIGRATE_PFN_WRITE; 1224 } 1225 migrate_device_pages(src_pfns, dst_pfns, npages); 1226 migrate_device_finalize(src_pfns, dst_pfns, npages); 1227 kvfree(src_pfns); 1228 kvfree(dst_pfns); 1229 } 1230 1231 /* Removes free pages from the free list so they can't be re-allocated */ 1232 static void dmirror_remove_free_pages(struct dmirror_chunk *devmem) 1233 { 1234 struct dmirror_device *mdevice = devmem->mdevice; 1235 struct page *page; 1236 1237 for (page = mdevice->free_pages; page; page = page->zone_device_data) 1238 if (dmirror_page_to_chunk(page) == devmem) 1239 mdevice->free_pages = page->zone_device_data; 1240 } 1241 1242 static void dmirror_device_remove_chunks(struct dmirror_device *mdevice) 1243 { 1244 unsigned int i; 1245 1246 mutex_lock(&mdevice->devmem_lock); 1247 if (mdevice->devmem_chunks) { 1248 for (i = 0; i < mdevice->devmem_count; i++) { 1249 struct dmirror_chunk *devmem = 1250 mdevice->devmem_chunks[i]; 1251 1252 spin_lock(&mdevice->lock); 1253 devmem->remove = true; 1254 dmirror_remove_free_pages(devmem); 1255 spin_unlock(&mdevice->lock); 1256 1257 dmirror_device_evict_chunk(devmem); 1258 memunmap_pages(&devmem->pagemap); 1259 if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 1260 release_mem_region(devmem->pagemap.range.start, 1261 range_len(&devmem->pagemap.range)); 1262 kfree(devmem); 1263 } 1264 mdevice->devmem_count = 0; 1265 mdevice->devmem_capacity = 0; 1266 mdevice->free_pages = NULL; 1267 kfree(mdevice->devmem_chunks); 1268 mdevice->devmem_chunks = NULL; 1269 } 1270 mutex_unlock(&mdevice->devmem_lock); 1271 } 1272 1273 static long dmirror_fops_unlocked_ioctl(struct file *filp, 1274 unsigned int command, 1275 unsigned long arg) 1276 { 1277 void __user *uarg = (void __user *)arg; 1278 struct hmm_dmirror_cmd cmd; 1279 struct dmirror *dmirror; 1280 int ret; 1281 1282 dmirror = filp->private_data; 1283 if (!dmirror) 1284 return -EINVAL; 1285 1286 if (copy_from_user(&cmd, uarg, sizeof(cmd))) 1287 return -EFAULT; 1288 1289 if (cmd.addr & ~PAGE_MASK) 1290 return -EINVAL; 1291 if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) 1292 return -EINVAL; 1293 1294 cmd.cpages = 0; 1295 cmd.faults = 0; 1296 1297 switch (command) { 1298 case HMM_DMIRROR_READ: 1299 ret = dmirror_read(dmirror, &cmd); 1300 break; 1301 1302 case HMM_DMIRROR_WRITE: 1303 ret = dmirror_write(dmirror, &cmd); 1304 break; 1305 1306 case HMM_DMIRROR_MIGRATE_TO_DEV: 1307 ret = dmirror_migrate_to_device(dmirror, &cmd); 1308 break; 1309 1310 case HMM_DMIRROR_MIGRATE_TO_SYS: 1311 ret = dmirror_migrate_to_system(dmirror, &cmd); 1312 break; 1313 1314 case HMM_DMIRROR_EXCLUSIVE: 1315 ret = dmirror_exclusive(dmirror, &cmd); 1316 break; 1317 1318 case HMM_DMIRROR_CHECK_EXCLUSIVE: 1319 ret = dmirror_check_atomic(dmirror, cmd.addr, 1320 cmd.addr + (cmd.npages << PAGE_SHIFT)); 1321 break; 1322 1323 case HMM_DMIRROR_SNAPSHOT: 1324 ret = dmirror_snapshot(dmirror, &cmd); 1325 break; 1326 1327 case HMM_DMIRROR_RELEASE: 1328 dmirror_device_remove_chunks(dmirror->mdevice); 1329 ret = 0; 1330 break; 1331 1332 default: 1333 return -EINVAL; 1334 } 1335 if (ret) 1336 return ret; 1337 1338 if (copy_to_user(uarg, &cmd, sizeof(cmd))) 1339 return -EFAULT; 1340 1341 return 0; 1342 } 1343 1344 static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma) 1345 { 1346 unsigned long addr; 1347 1348 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { 1349 struct page *page; 1350 int ret; 1351 1352 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1353 if (!page) 1354 return -ENOMEM; 1355 1356 ret = vm_insert_page(vma, addr, page); 1357 if (ret) { 1358 __free_page(page); 1359 return ret; 1360 } 1361 put_page(page); 1362 } 1363 1364 return 0; 1365 } 1366 1367 static const struct file_operations dmirror_fops = { 1368 .open = dmirror_fops_open, 1369 .release = dmirror_fops_release, 1370 .mmap = dmirror_fops_mmap, 1371 .unlocked_ioctl = dmirror_fops_unlocked_ioctl, 1372 .llseek = default_llseek, 1373 .owner = THIS_MODULE, 1374 }; 1375 1376 static void dmirror_devmem_free(struct page *page) 1377 { 1378 struct page *rpage = BACKING_PAGE(page); 1379 struct dmirror_device *mdevice; 1380 1381 if (rpage != page) 1382 __free_page(rpage); 1383 1384 mdevice = dmirror_page_to_device(page); 1385 spin_lock(&mdevice->lock); 1386 1387 /* Return page to our allocator if not freeing the chunk */ 1388 if (!dmirror_page_to_chunk(page)->remove) { 1389 mdevice->cfree++; 1390 page->zone_device_data = mdevice->free_pages; 1391 mdevice->free_pages = page; 1392 } 1393 spin_unlock(&mdevice->lock); 1394 } 1395 1396 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) 1397 { 1398 struct migrate_vma args = { 0 }; 1399 unsigned long src_pfns = 0; 1400 unsigned long dst_pfns = 0; 1401 struct page *rpage; 1402 struct dmirror *dmirror; 1403 vm_fault_t ret; 1404 1405 /* 1406 * Normally, a device would use the page->zone_device_data to point to 1407 * the mirror but here we use it to hold the page for the simulated 1408 * device memory and that page holds the pointer to the mirror. 1409 */ 1410 rpage = vmf->page->zone_device_data; 1411 dmirror = rpage->zone_device_data; 1412 1413 /* FIXME demonstrate how we can adjust migrate range */ 1414 args.vma = vmf->vma; 1415 args.start = vmf->address; 1416 args.end = args.start + PAGE_SIZE; 1417 args.src = &src_pfns; 1418 args.dst = &dst_pfns; 1419 args.pgmap_owner = dmirror->mdevice; 1420 args.flags = dmirror_select_device(dmirror); 1421 args.fault_page = vmf->page; 1422 1423 if (migrate_vma_setup(&args)) 1424 return VM_FAULT_SIGBUS; 1425 1426 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 1427 if (ret) 1428 return ret; 1429 migrate_vma_pages(&args); 1430 /* 1431 * No device finalize step is needed since 1432 * dmirror_devmem_fault_alloc_and_copy() will have already 1433 * invalidated the device page table. 1434 */ 1435 migrate_vma_finalize(&args); 1436 return 0; 1437 } 1438 1439 static const struct dev_pagemap_ops dmirror_devmem_ops = { 1440 .page_free = dmirror_devmem_free, 1441 .migrate_to_ram = dmirror_devmem_fault, 1442 }; 1443 1444 static int dmirror_device_init(struct dmirror_device *mdevice, int id) 1445 { 1446 dev_t dev; 1447 int ret; 1448 1449 dev = MKDEV(MAJOR(dmirror_dev), id); 1450 mutex_init(&mdevice->devmem_lock); 1451 spin_lock_init(&mdevice->lock); 1452 1453 cdev_init(&mdevice->cdevice, &dmirror_fops); 1454 mdevice->cdevice.owner = THIS_MODULE; 1455 device_initialize(&mdevice->device); 1456 mdevice->device.devt = dev; 1457 1458 ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id); 1459 if (ret) 1460 return ret; 1461 1462 ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); 1463 if (ret) 1464 return ret; 1465 1466 /* Build a list of free ZONE_DEVICE struct pages */ 1467 return dmirror_allocate_chunk(mdevice, NULL); 1468 } 1469 1470 static void dmirror_device_remove(struct dmirror_device *mdevice) 1471 { 1472 dmirror_device_remove_chunks(mdevice); 1473 cdev_device_del(&mdevice->cdevice, &mdevice->device); 1474 } 1475 1476 static int __init hmm_dmirror_init(void) 1477 { 1478 int ret; 1479 int id = 0; 1480 int ndevices = 0; 1481 1482 ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, 1483 "HMM_DMIRROR"); 1484 if (ret) 1485 goto err_unreg; 1486 1487 memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0])); 1488 dmirror_devices[ndevices++].zone_device_type = 1489 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1490 dmirror_devices[ndevices++].zone_device_type = 1491 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1492 if (spm_addr_dev0 && spm_addr_dev1) { 1493 dmirror_devices[ndevices++].zone_device_type = 1494 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1495 dmirror_devices[ndevices++].zone_device_type = 1496 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1497 } 1498 for (id = 0; id < ndevices; id++) { 1499 ret = dmirror_device_init(dmirror_devices + id, id); 1500 if (ret) 1501 goto err_chrdev; 1502 } 1503 1504 pr_info("HMM test module loaded. This is only for testing HMM.\n"); 1505 return 0; 1506 1507 err_chrdev: 1508 while (--id >= 0) 1509 dmirror_device_remove(dmirror_devices + id); 1510 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1511 err_unreg: 1512 return ret; 1513 } 1514 1515 static void __exit hmm_dmirror_exit(void) 1516 { 1517 int id; 1518 1519 for (id = 0; id < DMIRROR_NDEVICES; id++) 1520 if (dmirror_devices[id].zone_device_type) 1521 dmirror_device_remove(dmirror_devices + id); 1522 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1523 } 1524 1525 module_init(hmm_dmirror_init); 1526 module_exit(hmm_dmirror_exit); 1527 MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module"); 1528 MODULE_LICENSE("GPL"); 1529