83 	struct dmirror			*dmirror;  member
90 struct dmirror {  struct
147 dmirror_select_device(struct dmirror *dmirror)  in dmirror_select_device()  argument
149 	return (dmirror->mdevice->zone_device_type ==  in dmirror_select_device()
163 	struct dmirror *dmirror;  in dmirror_fops_open()  local
167 	dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);  in dmirror_fops_open()
168 	if (dmirror == NULL)  in dmirror_fops_open()
171 	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);  in dmirror_fops_open()
172 	mutex_init(&dmirror->mutex);  in dmirror_fops_open()
173 	xa_init(&dmirror->pt);  in dmirror_fops_open()
175 	ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,  in dmirror_fops_open()
178 		kfree(dmirror);  in dmirror_fops_open()
182 	filp->private_data = dmirror;  in dmirror_fops_open()
188 	struct dmirror *dmirror = filp->private_data;  in dmirror_fops_release()  local
190 	mmu_interval_notifier_remove(&dmirror->notifier);  in dmirror_fops_release()
191 	xa_destroy(&dmirror->pt);  in dmirror_fops_release()
192 	kfree(dmirror);  in dmirror_fops_release()
208 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)  in dmirror_do_fault()  argument
234 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);  in dmirror_do_fault()
242 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,  in dmirror_do_update()  argument
253 	xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,  in dmirror_do_update()
255 		xa_erase(&dmirror->pt, pfn);  in dmirror_do_update()
262 	struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);  in dmirror_interval_invalidate()  local
269 	    range->owner == dmirror->mdevice)  in dmirror_interval_invalidate()
273 		mutex_lock(&dmirror->mutex);  in dmirror_interval_invalidate()
274 	else if (!mutex_trylock(&dmirror->mutex))  in dmirror_interval_invalidate()
278 	dmirror_do_update(dmirror, range->start, range->end);  in dmirror_interval_invalidate()
280 	mutex_unlock(&dmirror->mutex);  in dmirror_interval_invalidate()
288 static int dmirror_range_fault(struct dmirror *dmirror,  in dmirror_range_fault()  argument
291 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_range_fault()
312 		mutex_lock(&dmirror->mutex);  in dmirror_range_fault()
315 			mutex_unlock(&dmirror->mutex);  in dmirror_range_fault()
321 	ret = dmirror_do_fault(dmirror, range);  in dmirror_range_fault()
323 	mutex_unlock(&dmirror->mutex);  in dmirror_range_fault()
328 static int dmirror_fault(struct dmirror *dmirror, unsigned long start,  in dmirror_fault()  argument
331 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_fault()
335 		.notifier = &dmirror->notifier,  in dmirror_fault()
340 		.dev_private_owner = dmirror->mdevice,  in dmirror_fault()
352 		ret = dmirror_range_fault(dmirror, &range);  in dmirror_fault()
361 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,  in dmirror_do_read()  argument
373 		entry = xa_load(&dmirror->pt, pfn);  in dmirror_do_read()
387 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)  in dmirror_read()  argument
404 		mutex_lock(&dmirror->mutex);  in dmirror_read()
405 		ret = dmirror_do_read(dmirror, start, end, &bounce);  in dmirror_read()
406 		mutex_unlock(&dmirror->mutex);  in dmirror_read()
411 		ret = dmirror_fault(dmirror, start, end, false);  in dmirror_read()
427 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,  in dmirror_do_write()  argument
439 		entry = xa_load(&dmirror->pt, pfn);  in dmirror_do_write()
453 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)  in dmirror_write()  argument
475 		mutex_lock(&dmirror->mutex);  in dmirror_write()
476 		ret = dmirror_do_write(dmirror, start, end, &bounce);  in dmirror_write()
477 		mutex_unlock(&dmirror->mutex);  in dmirror_write()
482 		ret = dmirror_fault(dmirror, start, end, true);  in dmirror_write()
641 					   struct dmirror *dmirror)  in dmirror_migrate_alloc_and_copy()  argument
643 	struct dmirror_device *mdevice = dmirror->mdevice;  in dmirror_migrate_alloc_and_copy()
683 		rpage->zone_device_data = dmirror;  in dmirror_migrate_alloc_and_copy()
694 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,  in dmirror_check_atomic()  argument
702 		entry = xa_load(&dmirror->pt, pfn);  in dmirror_check_atomic()
711 		struct dmirror *dmirror)  in dmirror_atomic_map()  argument
716 	mutex_lock(&dmirror->mutex);  in dmirror_atomic_map()
719 	entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC);  in dmirror_atomic_map()
721 		mutex_unlock(&dmirror->mutex);  in dmirror_atomic_map()
725 	mutex_unlock(&dmirror->mutex);  in dmirror_atomic_map()
730 					    struct dmirror *dmirror)  in dmirror_migrate_finalize_and_map()  argument
739 	mutex_lock(&dmirror->mutex);  in dmirror_migrate_finalize_and_map()
756 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);  in dmirror_migrate_finalize_and_map()
758 			mutex_unlock(&dmirror->mutex);  in dmirror_migrate_finalize_and_map()
763 	mutex_unlock(&dmirror->mutex);  in dmirror_migrate_finalize_and_map()
767 static int dmirror_exclusive(struct dmirror *dmirror,  in dmirror_exclusive()  argument
772 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_exclusive()
796 		ret = dmirror_atomic_map(addr, page, dmirror);  in dmirror_exclusive()
810 	mutex_lock(&dmirror->mutex);  in dmirror_exclusive()
811 	ret = dmirror_do_read(dmirror, start, end, &bounce);  in dmirror_exclusive()
812 	mutex_unlock(&dmirror->mutex);  in dmirror_exclusive()
825 						      struct dmirror *dmirror)  in dmirror_devmem_fault_alloc_and_copy()  argument
852 		xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);  in dmirror_devmem_fault_alloc_and_copy()
875 static int dmirror_migrate_to_system(struct dmirror *dmirror,  in dmirror_migrate_to_system()  argument
880 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_migrate_to_system()
914 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_system()
915 		args.flags = dmirror_select_device(dmirror);  in dmirror_migrate_to_system()
922 		dmirror_devmem_fault_alloc_and_copy(&args, dmirror);  in dmirror_migrate_to_system()
935 static int dmirror_migrate_to_device(struct dmirror *dmirror,  in dmirror_migrate_to_device()  argument
940 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_migrate_to_device()
974 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_device()
981 		dmirror_migrate_alloc_and_copy(&args, dmirror);  in dmirror_migrate_to_device()
983 		dmirror_migrate_finalize_and_map(&args, dmirror);  in dmirror_migrate_to_device()
996 	mutex_lock(&dmirror->mutex);  in dmirror_migrate_to_device()
997 	ret = dmirror_do_read(dmirror, start, end, &bounce);  in dmirror_migrate_to_device()
998 	mutex_unlock(&dmirror->mutex);  in dmirror_migrate_to_device()
1014 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,  in dmirror_mkentry()  argument
1031 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1037 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1061 	struct dmirror *dmirror = dmi->dmirror;  in dmirror_snapshot_invalidate()  local
1064 		mutex_lock(&dmirror->mutex);  in dmirror_snapshot_invalidate()
1065 	else if (!mutex_trylock(&dmirror->mutex))  in dmirror_snapshot_invalidate()
1074 	mutex_unlock(&dmirror->mutex);  in dmirror_snapshot_invalidate()
1082 static int dmirror_range_snapshot(struct dmirror *dmirror,  in dmirror_range_snapshot()  argument
1086 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_range_snapshot()
1094 	notifier.dmirror = dmirror;  in dmirror_range_snapshot()
1120 		mutex_lock(&dmirror->mutex);  in dmirror_range_snapshot()
1123 			mutex_unlock(&dmirror->mutex);  in dmirror_range_snapshot()
1131 		dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);  in dmirror_range_snapshot()
1133 	mutex_unlock(&dmirror->mutex);  in dmirror_range_snapshot()
1139 static int dmirror_snapshot(struct dmirror *dmirror,  in dmirror_snapshot()  argument
1142 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_snapshot()
1152 		.dev_private_owner = dmirror->mdevice,  in dmirror_snapshot()
1177 		ret = dmirror_range_snapshot(dmirror, &range, perm);  in dmirror_snapshot()
1280 	struct dmirror *dmirror;  in dmirror_fops_unlocked_ioctl()  local
1283 	dmirror = filp->private_data;  in dmirror_fops_unlocked_ioctl()
1284 	if (!dmirror)  in dmirror_fops_unlocked_ioctl()
1300 		ret = dmirror_read(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1304 		ret = dmirror_write(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1308 		ret = dmirror_migrate_to_device(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1312 		ret = dmirror_migrate_to_system(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1316 		ret = dmirror_exclusive(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1320 		ret = dmirror_check_atomic(dmirror, cmd.addr,  in dmirror_fops_unlocked_ioctl()
1325 		ret = dmirror_snapshot(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1329 		dmirror_device_remove_chunks(dmirror->mdevice);  in dmirror_fops_unlocked_ioctl()
1403 	struct dmirror *dmirror;  in dmirror_devmem_fault()  local
1412 	dmirror = rpage->zone_device_data;  in dmirror_devmem_fault()
1420 	args.pgmap_owner = dmirror->mdevice;  in dmirror_devmem_fault()
1421 	args.flags = dmirror_select_device(dmirror);  in dmirror_devmem_fault()
1427 	ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);  in dmirror_devmem_fault()