xref: /linux-6.15/drivers/md/dm-writecache.c (revision 2ee73ef6)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2018 Red Hat. All rights reserved.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include <linux/device-mapper.h>
9 #include <linux/module.h>
10 #include <linux/init.h>
11 #include <linux/vmalloc.h>
12 #include <linux/kthread.h>
13 #include <linux/dm-io.h>
14 #include <linux/dm-kcopyd.h>
15 #include <linux/dax.h>
16 #include <linux/pfn_t.h>
17 #include <linux/libnvdimm.h>
18 #include <linux/delay.h>
19 #include "dm-io-tracker.h"
20 
21 #define DM_MSG_PREFIX "writecache"
22 
23 #define HIGH_WATERMARK			50
24 #define LOW_WATERMARK			45
25 #define MAX_WRITEBACK_JOBS		min(0x10000000 / PAGE_SIZE, totalram_pages() / 16)
26 #define ENDIO_LATENCY			16
27 #define WRITEBACK_LATENCY		64
28 #define AUTOCOMMIT_BLOCKS_SSD		65536
29 #define AUTOCOMMIT_BLOCKS_PMEM		64
30 #define AUTOCOMMIT_MSEC			1000
31 #define MAX_AGE_DIV			16
32 #define MAX_AGE_UNSPECIFIED		-1UL
33 #define PAUSE_WRITEBACK			(HZ * 3)
34 
35 #define BITMAP_GRANULARITY	65536
36 #if BITMAP_GRANULARITY < PAGE_SIZE
37 #undef BITMAP_GRANULARITY
38 #define BITMAP_GRANULARITY	PAGE_SIZE
39 #endif
40 
41 #if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_FS_DAX)
42 #define DM_WRITECACHE_HAS_PMEM
43 #endif
44 
45 #ifdef DM_WRITECACHE_HAS_PMEM
46 #define pmem_assign(dest, src)					\
47 do {								\
48 	typeof(dest) uniq = (src);				\
49 	memcpy_flushcache(&(dest), &uniq, sizeof(dest));	\
50 } while (0)
51 #else
52 #define pmem_assign(dest, src)	((dest) = (src))
53 #endif
54 
55 #if IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC) && defined(DM_WRITECACHE_HAS_PMEM)
56 #define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
57 #endif
58 
59 #define MEMORY_SUPERBLOCK_MAGIC		0x23489321
60 #define MEMORY_SUPERBLOCK_VERSION	1
61 
62 struct wc_memory_entry {
63 	__le64 original_sector;
64 	__le64 seq_count;
65 };
66 
67 struct wc_memory_superblock {
68 	union {
69 		struct {
70 			__le32 magic;
71 			__le32 version;
72 			__le32 block_size;
73 			__le32 pad;
74 			__le64 n_blocks;
75 			__le64 seq_count;
76 		};
77 		__le64 padding[8];
78 	};
79 	struct wc_memory_entry entries[];
80 };
81 
82 struct wc_entry {
83 	struct rb_node rb_node;
84 	struct list_head lru;
85 	unsigned short wc_list_contiguous;
86 	bool write_in_progress
87 #if BITS_PER_LONG == 64
88 		:1
89 #endif
90 	;
91 	unsigned long index
92 #if BITS_PER_LONG == 64
93 		:47
94 #endif
95 	;
96 	unsigned long age;
97 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
98 	uint64_t original_sector;
99 	uint64_t seq_count;
100 #endif
101 };
102 
103 #ifdef DM_WRITECACHE_HAS_PMEM
104 #define WC_MODE_PMEM(wc)			((wc)->pmem_mode)
105 #define WC_MODE_FUA(wc)				((wc)->writeback_fua)
106 #else
107 #define WC_MODE_PMEM(wc)			false
108 #define WC_MODE_FUA(wc)				false
109 #endif
110 #define WC_MODE_SORT_FREELIST(wc)		(!WC_MODE_PMEM(wc))
111 
112 struct dm_writecache {
113 	struct mutex lock;
114 	struct list_head lru;
115 	union {
116 		struct list_head freelist;
117 		struct {
118 			struct rb_root freetree;
119 			struct wc_entry *current_free;
120 		};
121 	};
122 	struct rb_root tree;
123 
124 	size_t freelist_size;
125 	size_t writeback_size;
126 	size_t freelist_high_watermark;
127 	size_t freelist_low_watermark;
128 	unsigned long max_age;
129 	unsigned long pause;
130 
131 	unsigned uncommitted_blocks;
132 	unsigned autocommit_blocks;
133 	unsigned max_writeback_jobs;
134 
135 	int error;
136 
137 	unsigned long autocommit_jiffies;
138 	struct timer_list autocommit_timer;
139 	struct wait_queue_head freelist_wait;
140 
141 	struct timer_list max_age_timer;
142 
143 	atomic_t bio_in_progress[2];
144 	struct wait_queue_head bio_in_progress_wait[2];
145 
146 	struct dm_target *ti;
147 	struct dm_dev *dev;
148 	struct dm_dev *ssd_dev;
149 	sector_t start_sector;
150 	void *memory_map;
151 	uint64_t memory_map_size;
152 	size_t metadata_sectors;
153 	size_t n_blocks;
154 	uint64_t seq_count;
155 	sector_t data_device_sectors;
156 	void *block_start;
157 	struct wc_entry *entries;
158 	unsigned block_size;
159 	unsigned char block_size_bits;
160 
161 	bool pmem_mode:1;
162 	bool writeback_fua:1;
163 
164 	bool overwrote_committed:1;
165 	bool memory_vmapped:1;
166 
167 	bool start_sector_set:1;
168 	bool high_wm_percent_set:1;
169 	bool low_wm_percent_set:1;
170 	bool max_writeback_jobs_set:1;
171 	bool autocommit_blocks_set:1;
172 	bool autocommit_time_set:1;
173 	bool max_age_set:1;
174 	bool writeback_fua_set:1;
175 	bool flush_on_suspend:1;
176 	bool cleaner:1;
177 	bool cleaner_set:1;
178 	bool metadata_only:1;
179 	bool pause_set:1;
180 
181 	unsigned high_wm_percent_value;
182 	unsigned low_wm_percent_value;
183 	unsigned autocommit_time_value;
184 	unsigned max_age_value;
185 	unsigned pause_value;
186 
187 	unsigned writeback_all;
188 	struct workqueue_struct *writeback_wq;
189 	struct work_struct writeback_work;
190 	struct work_struct flush_work;
191 
192 	struct dm_io_tracker iot;
193 
194 	struct dm_io_client *dm_io;
195 
196 	raw_spinlock_t endio_list_lock;
197 	struct list_head endio_list;
198 	struct task_struct *endio_thread;
199 
200 	struct task_struct *flush_thread;
201 	struct bio_list flush_list;
202 
203 	struct dm_kcopyd_client *dm_kcopyd;
204 	unsigned long *dirty_bitmap;
205 	unsigned dirty_bitmap_size;
206 
207 	struct bio_set bio_set;
208 	mempool_t copy_pool;
209 
210 	struct {
211 		unsigned long long reads;
212 		unsigned long long read_hits;
213 		unsigned long long writes;
214 		unsigned long long write_hits_uncommitted;
215 		unsigned long long write_hits_committed;
216 		unsigned long long writes_around;
217 		unsigned long long writes_allocate;
218 		unsigned long long writes_blocked_on_freelist;
219 		unsigned long long flushes;
220 		unsigned long long discards;
221 	} stats;
222 };
223 
224 #define WB_LIST_INLINE		16
225 
226 struct writeback_struct {
227 	struct list_head endio_entry;
228 	struct dm_writecache *wc;
229 	struct wc_entry **wc_list;
230 	unsigned wc_list_n;
231 	struct wc_entry *wc_list_inline[WB_LIST_INLINE];
232 	struct bio bio;
233 };
234 
235 struct copy_struct {
236 	struct list_head endio_entry;
237 	struct dm_writecache *wc;
238 	struct wc_entry *e;
239 	unsigned n_entries;
240 	int error;
241 };
242 
243 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle,
244 					    "A percentage of time allocated for data copying");
245 
246 static void wc_lock(struct dm_writecache *wc)
247 {
248 	mutex_lock(&wc->lock);
249 }
250 
251 static void wc_unlock(struct dm_writecache *wc)
252 {
253 	mutex_unlock(&wc->lock);
254 }
255 
256 #ifdef DM_WRITECACHE_HAS_PMEM
257 static int persistent_memory_claim(struct dm_writecache *wc)
258 {
259 	int r;
260 	loff_t s;
261 	long p, da;
262 	pfn_t pfn;
263 	int id;
264 	struct page **pages;
265 	sector_t offset;
266 
267 	wc->memory_vmapped = false;
268 
269 	s = wc->memory_map_size;
270 	p = s >> PAGE_SHIFT;
271 	if (!p) {
272 		r = -EINVAL;
273 		goto err1;
274 	}
275 	if (p != s >> PAGE_SHIFT) {
276 		r = -EOVERFLOW;
277 		goto err1;
278 	}
279 
280 	offset = get_start_sect(wc->ssd_dev->bdev);
281 	if (offset & (PAGE_SIZE / 512 - 1)) {
282 		r = -EINVAL;
283 		goto err1;
284 	}
285 	offset >>= PAGE_SHIFT - 9;
286 
287 	id = dax_read_lock();
288 
289 	da = dax_direct_access(wc->ssd_dev->dax_dev, offset, p, DAX_ACCESS,
290 			&wc->memory_map, &pfn);
291 	if (da < 0) {
292 		wc->memory_map = NULL;
293 		r = da;
294 		goto err2;
295 	}
296 	if (!pfn_t_has_page(pfn)) {
297 		wc->memory_map = NULL;
298 		r = -EOPNOTSUPP;
299 		goto err2;
300 	}
301 	if (da != p) {
302 		long i;
303 		wc->memory_map = NULL;
304 		pages = kvmalloc_array(p, sizeof(struct page *), GFP_KERNEL);
305 		if (!pages) {
306 			r = -ENOMEM;
307 			goto err2;
308 		}
309 		i = 0;
310 		do {
311 			long daa;
312 			daa = dax_direct_access(wc->ssd_dev->dax_dev, offset + i,
313 					p - i, DAX_ACCESS, NULL, &pfn);
314 			if (daa <= 0) {
315 				r = daa ? daa : -EINVAL;
316 				goto err3;
317 			}
318 			if (!pfn_t_has_page(pfn)) {
319 				r = -EOPNOTSUPP;
320 				goto err3;
321 			}
322 			while (daa-- && i < p) {
323 				pages[i++] = pfn_t_to_page(pfn);
324 				pfn.val++;
325 				if (!(i & 15))
326 					cond_resched();
327 			}
328 		} while (i < p);
329 		wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL);
330 		if (!wc->memory_map) {
331 			r = -ENOMEM;
332 			goto err3;
333 		}
334 		kvfree(pages);
335 		wc->memory_vmapped = true;
336 	}
337 
338 	dax_read_unlock(id);
339 
340 	wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT;
341 	wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT;
342 
343 	return 0;
344 err3:
345 	kvfree(pages);
346 err2:
347 	dax_read_unlock(id);
348 err1:
349 	return r;
350 }
351 #else
352 static int persistent_memory_claim(struct dm_writecache *wc)
353 {
354 	return -EOPNOTSUPP;
355 }
356 #endif
357 
358 static void persistent_memory_release(struct dm_writecache *wc)
359 {
360 	if (wc->memory_vmapped)
361 		vunmap(wc->memory_map - ((size_t)wc->start_sector << SECTOR_SHIFT));
362 }
363 
364 static struct page *persistent_memory_page(void *addr)
365 {
366 	if (is_vmalloc_addr(addr))
367 		return vmalloc_to_page(addr);
368 	else
369 		return virt_to_page(addr);
370 }
371 
372 static unsigned persistent_memory_page_offset(void *addr)
373 {
374 	return (unsigned long)addr & (PAGE_SIZE - 1);
375 }
376 
377 static void persistent_memory_flush_cache(void *ptr, size_t size)
378 {
379 	if (is_vmalloc_addr(ptr))
380 		flush_kernel_vmap_range(ptr, size);
381 }
382 
383 static void persistent_memory_invalidate_cache(void *ptr, size_t size)
384 {
385 	if (is_vmalloc_addr(ptr))
386 		invalidate_kernel_vmap_range(ptr, size);
387 }
388 
389 static struct wc_memory_superblock *sb(struct dm_writecache *wc)
390 {
391 	return wc->memory_map;
392 }
393 
394 static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e)
395 {
396 	return &sb(wc)->entries[e->index];
397 }
398 
399 static void *memory_data(struct dm_writecache *wc, struct wc_entry *e)
400 {
401 	return (char *)wc->block_start + (e->index << wc->block_size_bits);
402 }
403 
404 static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e)
405 {
406 	return wc->start_sector + wc->metadata_sectors +
407 		((sector_t)e->index << (wc->block_size_bits - SECTOR_SHIFT));
408 }
409 
410 static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e)
411 {
412 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
413 	return e->original_sector;
414 #else
415 	return le64_to_cpu(memory_entry(wc, e)->original_sector);
416 #endif
417 }
418 
419 static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e)
420 {
421 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
422 	return e->seq_count;
423 #else
424 	return le64_to_cpu(memory_entry(wc, e)->seq_count);
425 #endif
426 }
427 
428 static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e)
429 {
430 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
431 	e->seq_count = -1;
432 #endif
433 	pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1));
434 }
435 
436 static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e,
437 					    uint64_t original_sector, uint64_t seq_count)
438 {
439 	struct wc_memory_entry me;
440 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
441 	e->original_sector = original_sector;
442 	e->seq_count = seq_count;
443 #endif
444 	me.original_sector = cpu_to_le64(original_sector);
445 	me.seq_count = cpu_to_le64(seq_count);
446 	pmem_assign(*memory_entry(wc, e), me);
447 }
448 
449 #define writecache_error(wc, err, msg, arg...)				\
450 do {									\
451 	if (!cmpxchg(&(wc)->error, 0, err))				\
452 		DMERR(msg, ##arg);					\
453 	wake_up(&(wc)->freelist_wait);					\
454 } while (0)
455 
456 #define writecache_has_error(wc)	(unlikely(READ_ONCE((wc)->error)))
457 
458 static void writecache_flush_all_metadata(struct dm_writecache *wc)
459 {
460 	if (!WC_MODE_PMEM(wc))
461 		memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size);
462 }
463 
464 static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size)
465 {
466 	if (!WC_MODE_PMEM(wc))
467 		__set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY,
468 			  wc->dirty_bitmap);
469 }
470 
471 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev);
472 
473 struct io_notify {
474 	struct dm_writecache *wc;
475 	struct completion c;
476 	atomic_t count;
477 };
478 
479 static void writecache_notify_io(unsigned long error, void *context)
480 {
481 	struct io_notify *endio = context;
482 
483 	if (unlikely(error != 0))
484 		writecache_error(endio->wc, -EIO, "error writing metadata");
485 	BUG_ON(atomic_read(&endio->count) <= 0);
486 	if (atomic_dec_and_test(&endio->count))
487 		complete(&endio->c);
488 }
489 
490 static void writecache_wait_for_ios(struct dm_writecache *wc, int direction)
491 {
492 	wait_event(wc->bio_in_progress_wait[direction],
493 		   !atomic_read(&wc->bio_in_progress[direction]));
494 }
495 
496 static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
497 {
498 	struct dm_io_region region;
499 	struct dm_io_request req;
500 	struct io_notify endio = {
501 		wc,
502 		COMPLETION_INITIALIZER_ONSTACK(endio.c),
503 		ATOMIC_INIT(1),
504 	};
505 	unsigned bitmap_bits = wc->dirty_bitmap_size * 8;
506 	unsigned i = 0;
507 
508 	while (1) {
509 		unsigned j;
510 		i = find_next_bit(wc->dirty_bitmap, bitmap_bits, i);
511 		if (unlikely(i == bitmap_bits))
512 			break;
513 		j = find_next_zero_bit(wc->dirty_bitmap, bitmap_bits, i);
514 
515 		region.bdev = wc->ssd_dev->bdev;
516 		region.sector = (sector_t)i * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
517 		region.count = (sector_t)(j - i) * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
518 
519 		if (unlikely(region.sector >= wc->metadata_sectors))
520 			break;
521 		if (unlikely(region.sector + region.count > wc->metadata_sectors))
522 			region.count = wc->metadata_sectors - region.sector;
523 
524 		region.sector += wc->start_sector;
525 		atomic_inc(&endio.count);
526 		req.bi_op = REQ_OP_WRITE;
527 		req.bi_op_flags = REQ_SYNC;
528 		req.mem.type = DM_IO_VMA;
529 		req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY;
530 		req.client = wc->dm_io;
531 		req.notify.fn = writecache_notify_io;
532 		req.notify.context = &endio;
533 
534 		/* writing via async dm-io (implied by notify.fn above) won't return an error */
535 	        (void) dm_io(&req, 1, &region, NULL);
536 		i = j;
537 	}
538 
539 	writecache_notify_io(0, &endio);
540 	wait_for_completion_io(&endio.c);
541 
542 	if (wait_for_ios)
543 		writecache_wait_for_ios(wc, WRITE);
544 
545 	writecache_disk_flush(wc, wc->ssd_dev);
546 
547 	memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size);
548 }
549 
550 static void ssd_commit_superblock(struct dm_writecache *wc)
551 {
552 	int r;
553 	struct dm_io_region region;
554 	struct dm_io_request req;
555 
556 	region.bdev = wc->ssd_dev->bdev;
557 	region.sector = 0;
558 	region.count = max(4096U, wc->block_size) >> SECTOR_SHIFT;
559 
560 	if (unlikely(region.sector + region.count > wc->metadata_sectors))
561 		region.count = wc->metadata_sectors - region.sector;
562 
563 	region.sector += wc->start_sector;
564 
565 	req.bi_op = REQ_OP_WRITE;
566 	req.bi_op_flags = REQ_SYNC | REQ_FUA;
567 	req.mem.type = DM_IO_VMA;
568 	req.mem.ptr.vma = (char *)wc->memory_map;
569 	req.client = wc->dm_io;
570 	req.notify.fn = NULL;
571 	req.notify.context = NULL;
572 
573 	r = dm_io(&req, 1, &region, NULL);
574 	if (unlikely(r))
575 		writecache_error(wc, r, "error writing superblock");
576 }
577 
578 static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
579 {
580 	if (WC_MODE_PMEM(wc))
581 		pmem_wmb();
582 	else
583 		ssd_commit_flushed(wc, wait_for_ios);
584 }
585 
586 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
587 {
588 	int r;
589 	struct dm_io_region region;
590 	struct dm_io_request req;
591 
592 	region.bdev = dev->bdev;
593 	region.sector = 0;
594 	region.count = 0;
595 	req.bi_op = REQ_OP_WRITE;
596 	req.bi_op_flags = REQ_PREFLUSH;
597 	req.mem.type = DM_IO_KMEM;
598 	req.mem.ptr.addr = NULL;
599 	req.client = wc->dm_io;
600 	req.notify.fn = NULL;
601 
602 	r = dm_io(&req, 1, &region, NULL);
603 	if (unlikely(r))
604 		writecache_error(wc, r, "error flushing metadata: %d", r);
605 }
606 
607 #define WFE_RETURN_FOLLOWING	1
608 #define WFE_LOWEST_SEQ		2
609 
610 static struct wc_entry *writecache_find_entry(struct dm_writecache *wc,
611 					      uint64_t block, int flags)
612 {
613 	struct wc_entry *e;
614 	struct rb_node *node = wc->tree.rb_node;
615 
616 	if (unlikely(!node))
617 		return NULL;
618 
619 	while (1) {
620 		e = container_of(node, struct wc_entry, rb_node);
621 		if (read_original_sector(wc, e) == block)
622 			break;
623 
624 		node = (read_original_sector(wc, e) >= block ?
625 			e->rb_node.rb_left : e->rb_node.rb_right);
626 		if (unlikely(!node)) {
627 			if (!(flags & WFE_RETURN_FOLLOWING))
628 				return NULL;
629 			if (read_original_sector(wc, e) >= block) {
630 				return e;
631 			} else {
632 				node = rb_next(&e->rb_node);
633 				if (unlikely(!node))
634 					return NULL;
635 				e = container_of(node, struct wc_entry, rb_node);
636 				return e;
637 			}
638 		}
639 	}
640 
641 	while (1) {
642 		struct wc_entry *e2;
643 		if (flags & WFE_LOWEST_SEQ)
644 			node = rb_prev(&e->rb_node);
645 		else
646 			node = rb_next(&e->rb_node);
647 		if (unlikely(!node))
648 			return e;
649 		e2 = container_of(node, struct wc_entry, rb_node);
650 		if (read_original_sector(wc, e2) != block)
651 			return e;
652 		e = e2;
653 	}
654 }
655 
656 static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *ins)
657 {
658 	struct wc_entry *e;
659 	struct rb_node **node = &wc->tree.rb_node, *parent = NULL;
660 
661 	while (*node) {
662 		e = container_of(*node, struct wc_entry, rb_node);
663 		parent = &e->rb_node;
664 		if (read_original_sector(wc, e) > read_original_sector(wc, ins))
665 			node = &parent->rb_left;
666 		else
667 			node = &parent->rb_right;
668 	}
669 	rb_link_node(&ins->rb_node, parent, node);
670 	rb_insert_color(&ins->rb_node, &wc->tree);
671 	list_add(&ins->lru, &wc->lru);
672 	ins->age = jiffies;
673 }
674 
675 static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
676 {
677 	list_del(&e->lru);
678 	rb_erase(&e->rb_node, &wc->tree);
679 }
680 
681 static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e)
682 {
683 	if (WC_MODE_SORT_FREELIST(wc)) {
684 		struct rb_node **node = &wc->freetree.rb_node, *parent = NULL;
685 		if (unlikely(!*node))
686 			wc->current_free = e;
687 		while (*node) {
688 			parent = *node;
689 			if (&e->rb_node < *node)
690 				node = &parent->rb_left;
691 			else
692 				node = &parent->rb_right;
693 		}
694 		rb_link_node(&e->rb_node, parent, node);
695 		rb_insert_color(&e->rb_node, &wc->freetree);
696 	} else {
697 		list_add_tail(&e->lru, &wc->freelist);
698 	}
699 	wc->freelist_size++;
700 }
701 
702 static inline void writecache_verify_watermark(struct dm_writecache *wc)
703 {
704 	if (unlikely(wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark))
705 		queue_work(wc->writeback_wq, &wc->writeback_work);
706 }
707 
708 static void writecache_max_age_timer(struct timer_list *t)
709 {
710 	struct dm_writecache *wc = from_timer(wc, t, max_age_timer);
711 
712 	if (!dm_suspended(wc->ti) && !writecache_has_error(wc)) {
713 		queue_work(wc->writeback_wq, &wc->writeback_work);
714 		mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
715 	}
716 }
717 
718 static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector)
719 {
720 	struct wc_entry *e;
721 
722 	if (WC_MODE_SORT_FREELIST(wc)) {
723 		struct rb_node *next;
724 		if (unlikely(!wc->current_free))
725 			return NULL;
726 		e = wc->current_free;
727 		if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
728 			return NULL;
729 		next = rb_next(&e->rb_node);
730 		rb_erase(&e->rb_node, &wc->freetree);
731 		if (unlikely(!next))
732 			next = rb_first(&wc->freetree);
733 		wc->current_free = next ? container_of(next, struct wc_entry, rb_node) : NULL;
734 	} else {
735 		if (unlikely(list_empty(&wc->freelist)))
736 			return NULL;
737 		e = container_of(wc->freelist.next, struct wc_entry, lru);
738 		if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
739 			return NULL;
740 		list_del(&e->lru);
741 	}
742 	wc->freelist_size--;
743 
744 	writecache_verify_watermark(wc);
745 
746 	return e;
747 }
748 
749 static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e)
750 {
751 	writecache_unlink(wc, e);
752 	writecache_add_to_freelist(wc, e);
753 	clear_seq_count(wc, e);
754 	writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
755 	if (unlikely(waitqueue_active(&wc->freelist_wait)))
756 		wake_up(&wc->freelist_wait);
757 }
758 
759 static void writecache_wait_on_freelist(struct dm_writecache *wc)
760 {
761 	DEFINE_WAIT(wait);
762 
763 	prepare_to_wait(&wc->freelist_wait, &wait, TASK_UNINTERRUPTIBLE);
764 	wc_unlock(wc);
765 	io_schedule();
766 	finish_wait(&wc->freelist_wait, &wait);
767 	wc_lock(wc);
768 }
769 
770 static void writecache_poison_lists(struct dm_writecache *wc)
771 {
772 	/*
773 	 * Catch incorrect access to these values while the device is suspended.
774 	 */
775 	memset(&wc->tree, -1, sizeof wc->tree);
776 	wc->lru.next = LIST_POISON1;
777 	wc->lru.prev = LIST_POISON2;
778 	wc->freelist.next = LIST_POISON1;
779 	wc->freelist.prev = LIST_POISON2;
780 }
781 
782 static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e)
783 {
784 	writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
785 	if (WC_MODE_PMEM(wc))
786 		writecache_flush_region(wc, memory_data(wc, e), wc->block_size);
787 }
788 
789 static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e)
790 {
791 	return read_seq_count(wc, e) < wc->seq_count;
792 }
793 
794 static void writecache_flush(struct dm_writecache *wc)
795 {
796 	struct wc_entry *e, *e2;
797 	bool need_flush_after_free;
798 
799 	wc->uncommitted_blocks = 0;
800 	del_timer(&wc->autocommit_timer);
801 
802 	if (list_empty(&wc->lru))
803 		return;
804 
805 	e = container_of(wc->lru.next, struct wc_entry, lru);
806 	if (writecache_entry_is_committed(wc, e)) {
807 		if (wc->overwrote_committed) {
808 			writecache_wait_for_ios(wc, WRITE);
809 			writecache_disk_flush(wc, wc->ssd_dev);
810 			wc->overwrote_committed = false;
811 		}
812 		return;
813 	}
814 	while (1) {
815 		writecache_flush_entry(wc, e);
816 		if (unlikely(e->lru.next == &wc->lru))
817 			break;
818 		e2 = container_of(e->lru.next, struct wc_entry, lru);
819 		if (writecache_entry_is_committed(wc, e2))
820 			break;
821 		e = e2;
822 		cond_resched();
823 	}
824 	writecache_commit_flushed(wc, true);
825 
826 	wc->seq_count++;
827 	pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count));
828 	if (WC_MODE_PMEM(wc))
829 		writecache_commit_flushed(wc, false);
830 	else
831 		ssd_commit_superblock(wc);
832 
833 	wc->overwrote_committed = false;
834 
835 	need_flush_after_free = false;
836 	while (1) {
837 		/* Free another committed entry with lower seq-count */
838 		struct rb_node *rb_node = rb_prev(&e->rb_node);
839 
840 		if (rb_node) {
841 			e2 = container_of(rb_node, struct wc_entry, rb_node);
842 			if (read_original_sector(wc, e2) == read_original_sector(wc, e) &&
843 			    likely(!e2->write_in_progress)) {
844 				writecache_free_entry(wc, e2);
845 				need_flush_after_free = true;
846 			}
847 		}
848 		if (unlikely(e->lru.prev == &wc->lru))
849 			break;
850 		e = container_of(e->lru.prev, struct wc_entry, lru);
851 		cond_resched();
852 	}
853 
854 	if (need_flush_after_free)
855 		writecache_commit_flushed(wc, false);
856 }
857 
858 static void writecache_flush_work(struct work_struct *work)
859 {
860 	struct dm_writecache *wc = container_of(work, struct dm_writecache, flush_work);
861 
862 	wc_lock(wc);
863 	writecache_flush(wc);
864 	wc_unlock(wc);
865 }
866 
867 static void writecache_autocommit_timer(struct timer_list *t)
868 {
869 	struct dm_writecache *wc = from_timer(wc, t, autocommit_timer);
870 	if (!writecache_has_error(wc))
871 		queue_work(wc->writeback_wq, &wc->flush_work);
872 }
873 
874 static void writecache_schedule_autocommit(struct dm_writecache *wc)
875 {
876 	if (!timer_pending(&wc->autocommit_timer))
877 		mod_timer(&wc->autocommit_timer, jiffies + wc->autocommit_jiffies);
878 }
879 
880 static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_t end)
881 {
882 	struct wc_entry *e;
883 	bool discarded_something = false;
884 
885 	e = writecache_find_entry(wc, start, WFE_RETURN_FOLLOWING | WFE_LOWEST_SEQ);
886 	if (unlikely(!e))
887 		return;
888 
889 	while (read_original_sector(wc, e) < end) {
890 		struct rb_node *node = rb_next(&e->rb_node);
891 
892 		if (likely(!e->write_in_progress)) {
893 			if (!discarded_something) {
894 				if (!WC_MODE_PMEM(wc)) {
895 					writecache_wait_for_ios(wc, READ);
896 					writecache_wait_for_ios(wc, WRITE);
897 				}
898 				discarded_something = true;
899 			}
900 			if (!writecache_entry_is_committed(wc, e))
901 				wc->uncommitted_blocks--;
902 			writecache_free_entry(wc, e);
903 		}
904 
905 		if (unlikely(!node))
906 			break;
907 
908 		e = container_of(node, struct wc_entry, rb_node);
909 	}
910 
911 	if (discarded_something)
912 		writecache_commit_flushed(wc, false);
913 }
914 
915 static bool writecache_wait_for_writeback(struct dm_writecache *wc)
916 {
917 	if (wc->writeback_size) {
918 		writecache_wait_on_freelist(wc);
919 		return true;
920 	}
921 	return false;
922 }
923 
924 static void writecache_suspend(struct dm_target *ti)
925 {
926 	struct dm_writecache *wc = ti->private;
927 	bool flush_on_suspend;
928 
929 	del_timer_sync(&wc->autocommit_timer);
930 	del_timer_sync(&wc->max_age_timer);
931 
932 	wc_lock(wc);
933 	writecache_flush(wc);
934 	flush_on_suspend = wc->flush_on_suspend;
935 	if (flush_on_suspend) {
936 		wc->flush_on_suspend = false;
937 		wc->writeback_all++;
938 		queue_work(wc->writeback_wq, &wc->writeback_work);
939 	}
940 	wc_unlock(wc);
941 
942 	drain_workqueue(wc->writeback_wq);
943 
944 	wc_lock(wc);
945 	if (flush_on_suspend)
946 		wc->writeback_all--;
947 	while (writecache_wait_for_writeback(wc));
948 
949 	if (WC_MODE_PMEM(wc))
950 		persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
951 
952 	writecache_poison_lists(wc);
953 
954 	wc_unlock(wc);
955 }
956 
957 static int writecache_alloc_entries(struct dm_writecache *wc)
958 {
959 	size_t b;
960 
961 	if (wc->entries)
962 		return 0;
963 	wc->entries = vmalloc(array_size(sizeof(struct wc_entry), wc->n_blocks));
964 	if (!wc->entries)
965 		return -ENOMEM;
966 	for (b = 0; b < wc->n_blocks; b++) {
967 		struct wc_entry *e = &wc->entries[b];
968 		e->index = b;
969 		e->write_in_progress = false;
970 		cond_resched();
971 	}
972 
973 	return 0;
974 }
975 
976 static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors)
977 {
978 	struct dm_io_region region;
979 	struct dm_io_request req;
980 
981 	region.bdev = wc->ssd_dev->bdev;
982 	region.sector = wc->start_sector;
983 	region.count = n_sectors;
984 	req.bi_op = REQ_OP_READ;
985 	req.bi_op_flags = REQ_SYNC;
986 	req.mem.type = DM_IO_VMA;
987 	req.mem.ptr.vma = (char *)wc->memory_map;
988 	req.client = wc->dm_io;
989 	req.notify.fn = NULL;
990 
991 	return dm_io(&req, 1, &region, NULL);
992 }
993 
994 static void writecache_resume(struct dm_target *ti)
995 {
996 	struct dm_writecache *wc = ti->private;
997 	size_t b;
998 	bool need_flush = false;
999 	__le64 sb_seq_count;
1000 	int r;
1001 
1002 	wc_lock(wc);
1003 
1004 	wc->data_device_sectors = bdev_nr_sectors(wc->dev->bdev);
1005 
1006 	if (WC_MODE_PMEM(wc)) {
1007 		persistent_memory_invalidate_cache(wc->memory_map, wc->memory_map_size);
1008 	} else {
1009 		r = writecache_read_metadata(wc, wc->metadata_sectors);
1010 		if (r) {
1011 			size_t sb_entries_offset;
1012 			writecache_error(wc, r, "unable to read metadata: %d", r);
1013 			sb_entries_offset = offsetof(struct wc_memory_superblock, entries);
1014 			memset((char *)wc->memory_map + sb_entries_offset, -1,
1015 			       (wc->metadata_sectors << SECTOR_SHIFT) - sb_entries_offset);
1016 		}
1017 	}
1018 
1019 	wc->tree = RB_ROOT;
1020 	INIT_LIST_HEAD(&wc->lru);
1021 	if (WC_MODE_SORT_FREELIST(wc)) {
1022 		wc->freetree = RB_ROOT;
1023 		wc->current_free = NULL;
1024 	} else {
1025 		INIT_LIST_HEAD(&wc->freelist);
1026 	}
1027 	wc->freelist_size = 0;
1028 
1029 	r = copy_mc_to_kernel(&sb_seq_count, &sb(wc)->seq_count,
1030 			      sizeof(uint64_t));
1031 	if (r) {
1032 		writecache_error(wc, r, "hardware memory error when reading superblock: %d", r);
1033 		sb_seq_count = cpu_to_le64(0);
1034 	}
1035 	wc->seq_count = le64_to_cpu(sb_seq_count);
1036 
1037 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
1038 	for (b = 0; b < wc->n_blocks; b++) {
1039 		struct wc_entry *e = &wc->entries[b];
1040 		struct wc_memory_entry wme;
1041 		if (writecache_has_error(wc)) {
1042 			e->original_sector = -1;
1043 			e->seq_count = -1;
1044 			continue;
1045 		}
1046 		r = copy_mc_to_kernel(&wme, memory_entry(wc, e),
1047 				      sizeof(struct wc_memory_entry));
1048 		if (r) {
1049 			writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d",
1050 					 (unsigned long)b, r);
1051 			e->original_sector = -1;
1052 			e->seq_count = -1;
1053 		} else {
1054 			e->original_sector = le64_to_cpu(wme.original_sector);
1055 			e->seq_count = le64_to_cpu(wme.seq_count);
1056 		}
1057 		cond_resched();
1058 	}
1059 #endif
1060 	for (b = 0; b < wc->n_blocks; b++) {
1061 		struct wc_entry *e = &wc->entries[b];
1062 		if (!writecache_entry_is_committed(wc, e)) {
1063 			if (read_seq_count(wc, e) != -1) {
1064 erase_this:
1065 				clear_seq_count(wc, e);
1066 				need_flush = true;
1067 			}
1068 			writecache_add_to_freelist(wc, e);
1069 		} else {
1070 			struct wc_entry *old;
1071 
1072 			old = writecache_find_entry(wc, read_original_sector(wc, e), 0);
1073 			if (!old) {
1074 				writecache_insert_entry(wc, e);
1075 			} else {
1076 				if (read_seq_count(wc, old) == read_seq_count(wc, e)) {
1077 					writecache_error(wc, -EINVAL,
1078 						 "two identical entries, position %llu, sector %llu, sequence %llu",
1079 						 (unsigned long long)b, (unsigned long long)read_original_sector(wc, e),
1080 						 (unsigned long long)read_seq_count(wc, e));
1081 				}
1082 				if (read_seq_count(wc, old) > read_seq_count(wc, e)) {
1083 					goto erase_this;
1084 				} else {
1085 					writecache_free_entry(wc, old);
1086 					writecache_insert_entry(wc, e);
1087 					need_flush = true;
1088 				}
1089 			}
1090 		}
1091 		cond_resched();
1092 	}
1093 
1094 	if (need_flush) {
1095 		writecache_flush_all_metadata(wc);
1096 		writecache_commit_flushed(wc, false);
1097 	}
1098 
1099 	writecache_verify_watermark(wc);
1100 
1101 	if (wc->max_age != MAX_AGE_UNSPECIFIED)
1102 		mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
1103 
1104 	wc_unlock(wc);
1105 }
1106 
1107 static int process_flush_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
1108 {
1109 	if (argc != 1)
1110 		return -EINVAL;
1111 
1112 	wc_lock(wc);
1113 	if (dm_suspended(wc->ti)) {
1114 		wc_unlock(wc);
1115 		return -EBUSY;
1116 	}
1117 	if (writecache_has_error(wc)) {
1118 		wc_unlock(wc);
1119 		return -EIO;
1120 	}
1121 
1122 	writecache_flush(wc);
1123 	wc->writeback_all++;
1124 	queue_work(wc->writeback_wq, &wc->writeback_work);
1125 	wc_unlock(wc);
1126 
1127 	flush_workqueue(wc->writeback_wq);
1128 
1129 	wc_lock(wc);
1130 	wc->writeback_all--;
1131 	if (writecache_has_error(wc)) {
1132 		wc_unlock(wc);
1133 		return -EIO;
1134 	}
1135 	wc_unlock(wc);
1136 
1137 	return 0;
1138 }
1139 
1140 static int process_flush_on_suspend_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
1141 {
1142 	if (argc != 1)
1143 		return -EINVAL;
1144 
1145 	wc_lock(wc);
1146 	wc->flush_on_suspend = true;
1147 	wc_unlock(wc);
1148 
1149 	return 0;
1150 }
1151 
1152 static void activate_cleaner(struct dm_writecache *wc)
1153 {
1154 	wc->flush_on_suspend = true;
1155 	wc->cleaner = true;
1156 	wc->freelist_high_watermark = wc->n_blocks;
1157 	wc->freelist_low_watermark = wc->n_blocks;
1158 }
1159 
1160 static int process_cleaner_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
1161 {
1162 	if (argc != 1)
1163 		return -EINVAL;
1164 
1165 	wc_lock(wc);
1166 	activate_cleaner(wc);
1167 	if (!dm_suspended(wc->ti))
1168 		writecache_verify_watermark(wc);
1169 	wc_unlock(wc);
1170 
1171 	return 0;
1172 }
1173 
1174 static int process_clear_stats_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
1175 {
1176 	if (argc != 1)
1177 		return -EINVAL;
1178 
1179 	wc_lock(wc);
1180 	memset(&wc->stats, 0, sizeof wc->stats);
1181 	wc_unlock(wc);
1182 
1183 	return 0;
1184 }
1185 
1186 static int writecache_message(struct dm_target *ti, unsigned argc, char **argv,
1187 			      char *result, unsigned maxlen)
1188 {
1189 	int r = -EINVAL;
1190 	struct dm_writecache *wc = ti->private;
1191 
1192 	if (!strcasecmp(argv[0], "flush"))
1193 		r = process_flush_mesg(argc, argv, wc);
1194 	else if (!strcasecmp(argv[0], "flush_on_suspend"))
1195 		r = process_flush_on_suspend_mesg(argc, argv, wc);
1196 	else if (!strcasecmp(argv[0], "cleaner"))
1197 		r = process_cleaner_mesg(argc, argv, wc);
1198 	else if (!strcasecmp(argv[0], "clear_stats"))
1199 		r = process_clear_stats_mesg(argc, argv, wc);
1200 	else
1201 		DMERR("unrecognised message received: %s", argv[0]);
1202 
1203 	return r;
1204 }
1205 
1206 static void memcpy_flushcache_optimized(void *dest, void *source, size_t size)
1207 {
1208 	/*
1209 	 * clflushopt performs better with block size 1024, 2048, 4096
1210 	 * non-temporal stores perform better with block size 512
1211 	 *
1212 	 * block size   512             1024            2048            4096
1213 	 * movnti       496 MB/s        642 MB/s        725 MB/s        744 MB/s
1214 	 * clflushopt   373 MB/s        688 MB/s        1.1 GB/s        1.2 GB/s
1215 	 *
1216 	 * We see that movnti performs better for 512-byte blocks, and
1217 	 * clflushopt performs better for 1024-byte and larger blocks. So, we
1218 	 * prefer clflushopt for sizes >= 768.
1219 	 *
1220 	 * NOTE: this happens to be the case now (with dm-writecache's single
1221 	 * threaded model) but re-evaluate this once memcpy_flushcache() is
1222 	 * enabled to use movdir64b which might invalidate this performance
1223 	 * advantage seen with cache-allocating-writes plus flushing.
1224 	 */
1225 #ifdef CONFIG_X86
1226 	if (static_cpu_has(X86_FEATURE_CLFLUSHOPT) &&
1227 	    likely(boot_cpu_data.x86_clflush_size == 64) &&
1228 	    likely(size >= 768)) {
1229 		do {
1230 			memcpy((void *)dest, (void *)source, 64);
1231 			clflushopt((void *)dest);
1232 			dest += 64;
1233 			source += 64;
1234 			size -= 64;
1235 		} while (size >= 64);
1236 		return;
1237 	}
1238 #endif
1239 	memcpy_flushcache(dest, source, size);
1240 }
1241 
1242 static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
1243 {
1244 	void *buf;
1245 	unsigned size;
1246 	int rw = bio_data_dir(bio);
1247 	unsigned remaining_size = wc->block_size;
1248 
1249 	do {
1250 		struct bio_vec bv = bio_iter_iovec(bio, bio->bi_iter);
1251 		buf = bvec_kmap_local(&bv);
1252 		size = bv.bv_len;
1253 		if (unlikely(size > remaining_size))
1254 			size = remaining_size;
1255 
1256 		if (rw == READ) {
1257 			int r;
1258 			r = copy_mc_to_kernel(buf, data, size);
1259 			flush_dcache_page(bio_page(bio));
1260 			if (unlikely(r)) {
1261 				writecache_error(wc, r, "hardware memory error when reading data: %d", r);
1262 				bio->bi_status = BLK_STS_IOERR;
1263 			}
1264 		} else {
1265 			flush_dcache_page(bio_page(bio));
1266 			memcpy_flushcache_optimized(data, buf, size);
1267 		}
1268 
1269 		kunmap_local(buf);
1270 
1271 		data = (char *)data + size;
1272 		remaining_size -= size;
1273 		bio_advance(bio, size);
1274 	} while (unlikely(remaining_size));
1275 }
1276 
1277 static int writecache_flush_thread(void *data)
1278 {
1279 	struct dm_writecache *wc = data;
1280 
1281 	while (1) {
1282 		struct bio *bio;
1283 
1284 		wc_lock(wc);
1285 		bio = bio_list_pop(&wc->flush_list);
1286 		if (!bio) {
1287 			set_current_state(TASK_INTERRUPTIBLE);
1288 			wc_unlock(wc);
1289 
1290 			if (unlikely(kthread_should_stop())) {
1291 				set_current_state(TASK_RUNNING);
1292 				break;
1293 			}
1294 
1295 			schedule();
1296 			continue;
1297 		}
1298 
1299 		if (bio_op(bio) == REQ_OP_DISCARD) {
1300 			writecache_discard(wc, bio->bi_iter.bi_sector,
1301 					   bio_end_sector(bio));
1302 			wc_unlock(wc);
1303 			bio_set_dev(bio, wc->dev->bdev);
1304 			submit_bio_noacct(bio);
1305 		} else {
1306 			writecache_flush(wc);
1307 			wc_unlock(wc);
1308 			if (writecache_has_error(wc))
1309 				bio->bi_status = BLK_STS_IOERR;
1310 			bio_endio(bio);
1311 		}
1312 	}
1313 
1314 	return 0;
1315 }
1316 
1317 static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio)
1318 {
1319 	if (bio_list_empty(&wc->flush_list))
1320 		wake_up_process(wc->flush_thread);
1321 	bio_list_add(&wc->flush_list, bio);
1322 }
1323 
1324 enum wc_map_op {
1325 	WC_MAP_SUBMIT,
1326 	WC_MAP_REMAP,
1327 	WC_MAP_REMAP_ORIGIN,
1328 	WC_MAP_RETURN,
1329 	WC_MAP_ERROR,
1330 };
1331 
1332 static void writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio,
1333 					struct wc_entry *e)
1334 {
1335 	if (e) {
1336 		sector_t next_boundary =
1337 			read_original_sector(wc, e) - bio->bi_iter.bi_sector;
1338 		if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT)
1339 			dm_accept_partial_bio(bio, next_boundary);
1340 	}
1341 }
1342 
1343 static enum wc_map_op writecache_map_read(struct dm_writecache *wc, struct bio *bio)
1344 {
1345 	enum wc_map_op map_op;
1346 	struct wc_entry *e;
1347 
1348 read_next_block:
1349 	wc->stats.reads++;
1350 	e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
1351 	if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) {
1352 		wc->stats.read_hits++;
1353 		if (WC_MODE_PMEM(wc)) {
1354 			bio_copy_block(wc, bio, memory_data(wc, e));
1355 			if (bio->bi_iter.bi_size)
1356 				goto read_next_block;
1357 			map_op = WC_MAP_SUBMIT;
1358 		} else {
1359 			dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
1360 			bio_set_dev(bio, wc->ssd_dev->bdev);
1361 			bio->bi_iter.bi_sector = cache_sector(wc, e);
1362 			if (!writecache_entry_is_committed(wc, e))
1363 				writecache_wait_for_ios(wc, WRITE);
1364 			map_op = WC_MAP_REMAP;
1365 		}
1366 	} else {
1367 		writecache_map_remap_origin(wc, bio, e);
1368 		wc->stats.reads += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
1369 		map_op = WC_MAP_REMAP_ORIGIN;
1370 	}
1371 
1372 	return map_op;
1373 }
1374 
1375 static void writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio,
1376 				    struct wc_entry *e, bool search_used)
1377 {
1378 	unsigned bio_size = wc->block_size;
1379 	sector_t start_cache_sec = cache_sector(wc, e);
1380 	sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT);
1381 
1382 	while (bio_size < bio->bi_iter.bi_size) {
1383 		if (!search_used) {
1384 			struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec);
1385 			if (!f)
1386 				break;
1387 			write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector +
1388 							(bio_size >> SECTOR_SHIFT), wc->seq_count);
1389 			writecache_insert_entry(wc, f);
1390 			wc->uncommitted_blocks++;
1391 		} else {
1392 			struct wc_entry *f;
1393 			struct rb_node *next = rb_next(&e->rb_node);
1394 			if (!next)
1395 				break;
1396 			f = container_of(next, struct wc_entry, rb_node);
1397 			if (f != e + 1)
1398 				break;
1399 			if (read_original_sector(wc, f) !=
1400 			    read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
1401 				break;
1402 			if (unlikely(f->write_in_progress))
1403 				break;
1404 			if (writecache_entry_is_committed(wc, f))
1405 				wc->overwrote_committed = true;
1406 			e = f;
1407 		}
1408 		bio_size += wc->block_size;
1409 		current_cache_sec += wc->block_size >> SECTOR_SHIFT;
1410 	}
1411 
1412 	bio_set_dev(bio, wc->ssd_dev->bdev);
1413 	bio->bi_iter.bi_sector = start_cache_sec;
1414 	dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
1415 
1416 	wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1417 	wc->stats.writes_allocate += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
1418 
1419 	if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
1420 		wc->uncommitted_blocks = 0;
1421 		queue_work(wc->writeback_wq, &wc->flush_work);
1422 	} else {
1423 		writecache_schedule_autocommit(wc);
1424 	}
1425 }
1426 
1427 static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio *bio)
1428 {
1429 	struct wc_entry *e;
1430 
1431 	do {
1432 		bool found_entry = false;
1433 		bool search_used = false;
1434 		if (writecache_has_error(wc)) {
1435 			wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1436 			return WC_MAP_ERROR;
1437 		}
1438 		e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
1439 		if (e) {
1440 			if (!writecache_entry_is_committed(wc, e)) {
1441 				wc->stats.write_hits_uncommitted++;
1442 				search_used = true;
1443 				goto bio_copy;
1444 			}
1445 			wc->stats.write_hits_committed++;
1446 			if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
1447 				wc->overwrote_committed = true;
1448 				search_used = true;
1449 				goto bio_copy;
1450 			}
1451 			found_entry = true;
1452 		} else {
1453 			if (unlikely(wc->cleaner) ||
1454 			    (wc->metadata_only && !(bio->bi_opf & REQ_META)))
1455 				goto direct_write;
1456 		}
1457 		e = writecache_pop_from_freelist(wc, (sector_t)-1);
1458 		if (unlikely(!e)) {
1459 			if (!WC_MODE_PMEM(wc) && !found_entry) {
1460 direct_write:
1461 				e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
1462 				writecache_map_remap_origin(wc, bio, e);
1463 				wc->stats.writes_around += bio->bi_iter.bi_size >> wc->block_size_bits;
1464 				wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1465 				return WC_MAP_REMAP_ORIGIN;
1466 			}
1467 			wc->stats.writes_blocked_on_freelist++;
1468 			writecache_wait_on_freelist(wc);
1469 			continue;
1470 		}
1471 		write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
1472 		writecache_insert_entry(wc, e);
1473 		wc->uncommitted_blocks++;
1474 		wc->stats.writes_allocate++;
1475 bio_copy:
1476 		if (WC_MODE_PMEM(wc)) {
1477 			bio_copy_block(wc, bio, memory_data(wc, e));
1478 			wc->stats.writes++;
1479 		} else {
1480 			writecache_bio_copy_ssd(wc, bio, e, search_used);
1481 			return WC_MAP_REMAP;
1482 		}
1483 	} while (bio->bi_iter.bi_size);
1484 
1485 	if (unlikely(bio->bi_opf & REQ_FUA || wc->uncommitted_blocks >= wc->autocommit_blocks))
1486 		writecache_flush(wc);
1487 	else
1488 		writecache_schedule_autocommit(wc);
1489 
1490 	return WC_MAP_SUBMIT;
1491 }
1492 
1493 static enum wc_map_op writecache_map_flush(struct dm_writecache *wc, struct bio *bio)
1494 {
1495 	if (writecache_has_error(wc))
1496 		return WC_MAP_ERROR;
1497 
1498 	if (WC_MODE_PMEM(wc)) {
1499 		wc->stats.flushes++;
1500 		writecache_flush(wc);
1501 		if (writecache_has_error(wc))
1502 			return WC_MAP_ERROR;
1503 		else if (unlikely(wc->cleaner) || unlikely(wc->metadata_only))
1504 			return WC_MAP_REMAP_ORIGIN;
1505 		return WC_MAP_SUBMIT;
1506 	}
1507 	/* SSD: */
1508 	if (dm_bio_get_target_bio_nr(bio))
1509 		return WC_MAP_REMAP_ORIGIN;
1510 	wc->stats.flushes++;
1511 	writecache_offload_bio(wc, bio);
1512 	return WC_MAP_RETURN;
1513 }
1514 
1515 static enum wc_map_op writecache_map_discard(struct dm_writecache *wc, struct bio *bio)
1516 {
1517 	wc->stats.discards += bio->bi_iter.bi_size >> wc->block_size_bits;
1518 
1519 	if (writecache_has_error(wc))
1520 		return WC_MAP_ERROR;
1521 
1522 	if (WC_MODE_PMEM(wc)) {
1523 		writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio));
1524 		return WC_MAP_REMAP_ORIGIN;
1525 	}
1526 	/* SSD: */
1527 	writecache_offload_bio(wc, bio);
1528 	return WC_MAP_RETURN;
1529 }
1530 
1531 static int writecache_map(struct dm_target *ti, struct bio *bio)
1532 {
1533 	struct dm_writecache *wc = ti->private;
1534 	enum wc_map_op map_op;
1535 
1536 	bio->bi_private = NULL;
1537 
1538 	wc_lock(wc);
1539 
1540 	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1541 		map_op = writecache_map_flush(wc, bio);
1542 		goto done;
1543 	}
1544 
1545 	bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1546 
1547 	if (unlikely((((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) &
1548 				(wc->block_size / 512 - 1)) != 0)) {
1549 		DMERR("I/O is not aligned, sector %llu, size %u, block size %u",
1550 		      (unsigned long long)bio->bi_iter.bi_sector,
1551 		      bio->bi_iter.bi_size, wc->block_size);
1552 		map_op = WC_MAP_ERROR;
1553 		goto done;
1554 	}
1555 
1556 	if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
1557 		map_op = writecache_map_discard(wc, bio);
1558 		goto done;
1559 	}
1560 
1561 	if (bio_data_dir(bio) == READ)
1562 		map_op = writecache_map_read(wc, bio);
1563 	else
1564 		map_op = writecache_map_write(wc, bio);
1565 done:
1566 	switch (map_op) {
1567 	case WC_MAP_REMAP_ORIGIN:
1568 		if (likely(wc->pause != 0)) {
1569 			if (bio_op(bio) == REQ_OP_WRITE) {
1570 				dm_iot_io_begin(&wc->iot, 1);
1571 				bio->bi_private = (void *)2;
1572 			}
1573 		}
1574 		bio_set_dev(bio, wc->dev->bdev);
1575 		wc_unlock(wc);
1576 		return DM_MAPIO_REMAPPED;
1577 
1578 	case WC_MAP_REMAP:
1579 		/* make sure that writecache_end_io decrements bio_in_progress: */
1580 		bio->bi_private = (void *)1;
1581 		atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]);
1582 		wc_unlock(wc);
1583 		return DM_MAPIO_REMAPPED;
1584 
1585 	case WC_MAP_SUBMIT:
1586 		wc_unlock(wc);
1587 		bio_endio(bio);
1588 		return DM_MAPIO_SUBMITTED;
1589 
1590 	case WC_MAP_RETURN:
1591 		wc_unlock(wc);
1592 		return DM_MAPIO_SUBMITTED;
1593 
1594 	case WC_MAP_ERROR:
1595 		wc_unlock(wc);
1596 		bio_io_error(bio);
1597 		return DM_MAPIO_SUBMITTED;
1598 
1599 	default:
1600 		BUG();
1601 		return -1;
1602 	}
1603 }
1604 
1605 static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
1606 {
1607 	struct dm_writecache *wc = ti->private;
1608 
1609 	if (bio->bi_private == (void *)1) {
1610 		int dir = bio_data_dir(bio);
1611 		if (atomic_dec_and_test(&wc->bio_in_progress[dir]))
1612 			if (unlikely(waitqueue_active(&wc->bio_in_progress_wait[dir])))
1613 				wake_up(&wc->bio_in_progress_wait[dir]);
1614 	} else if (bio->bi_private == (void *)2) {
1615 		dm_iot_io_end(&wc->iot, 1);
1616 	}
1617 	return 0;
1618 }
1619 
1620 static int writecache_iterate_devices(struct dm_target *ti,
1621 				      iterate_devices_callout_fn fn, void *data)
1622 {
1623 	struct dm_writecache *wc = ti->private;
1624 
1625 	return fn(ti, wc->dev, 0, ti->len, data);
1626 }
1627 
1628 static void writecache_io_hints(struct dm_target *ti, struct queue_limits *limits)
1629 {
1630 	struct dm_writecache *wc = ti->private;
1631 
1632 	if (limits->logical_block_size < wc->block_size)
1633 		limits->logical_block_size = wc->block_size;
1634 
1635 	if (limits->physical_block_size < wc->block_size)
1636 		limits->physical_block_size = wc->block_size;
1637 
1638 	if (limits->io_min < wc->block_size)
1639 		limits->io_min = wc->block_size;
1640 }
1641 
1642 
1643 static void writecache_writeback_endio(struct bio *bio)
1644 {
1645 	struct writeback_struct *wb = container_of(bio, struct writeback_struct, bio);
1646 	struct dm_writecache *wc = wb->wc;
1647 	unsigned long flags;
1648 
1649 	raw_spin_lock_irqsave(&wc->endio_list_lock, flags);
1650 	if (unlikely(list_empty(&wc->endio_list)))
1651 		wake_up_process(wc->endio_thread);
1652 	list_add_tail(&wb->endio_entry, &wc->endio_list);
1653 	raw_spin_unlock_irqrestore(&wc->endio_list_lock, flags);
1654 }
1655 
1656 static void writecache_copy_endio(int read_err, unsigned long write_err, void *ptr)
1657 {
1658 	struct copy_struct *c = ptr;
1659 	struct dm_writecache *wc = c->wc;
1660 
1661 	c->error = likely(!(read_err | write_err)) ? 0 : -EIO;
1662 
1663 	raw_spin_lock_irq(&wc->endio_list_lock);
1664 	if (unlikely(list_empty(&wc->endio_list)))
1665 		wake_up_process(wc->endio_thread);
1666 	list_add_tail(&c->endio_entry, &wc->endio_list);
1667 	raw_spin_unlock_irq(&wc->endio_list_lock);
1668 }
1669 
1670 static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *list)
1671 {
1672 	unsigned i;
1673 	struct writeback_struct *wb;
1674 	struct wc_entry *e;
1675 	unsigned long n_walked = 0;
1676 
1677 	do {
1678 		wb = list_entry(list->next, struct writeback_struct, endio_entry);
1679 		list_del(&wb->endio_entry);
1680 
1681 		if (unlikely(wb->bio.bi_status != BLK_STS_OK))
1682 			writecache_error(wc, blk_status_to_errno(wb->bio.bi_status),
1683 					"write error %d", wb->bio.bi_status);
1684 		i = 0;
1685 		do {
1686 			e = wb->wc_list[i];
1687 			BUG_ON(!e->write_in_progress);
1688 			e->write_in_progress = false;
1689 			INIT_LIST_HEAD(&e->lru);
1690 			if (!writecache_has_error(wc))
1691 				writecache_free_entry(wc, e);
1692 			BUG_ON(!wc->writeback_size);
1693 			wc->writeback_size--;
1694 			n_walked++;
1695 			if (unlikely(n_walked >= ENDIO_LATENCY)) {
1696 				writecache_commit_flushed(wc, false);
1697 				wc_unlock(wc);
1698 				wc_lock(wc);
1699 				n_walked = 0;
1700 			}
1701 		} while (++i < wb->wc_list_n);
1702 
1703 		if (wb->wc_list != wb->wc_list_inline)
1704 			kfree(wb->wc_list);
1705 		bio_put(&wb->bio);
1706 	} while (!list_empty(list));
1707 }
1708 
1709 static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *list)
1710 {
1711 	struct copy_struct *c;
1712 	struct wc_entry *e;
1713 
1714 	do {
1715 		c = list_entry(list->next, struct copy_struct, endio_entry);
1716 		list_del(&c->endio_entry);
1717 
1718 		if (unlikely(c->error))
1719 			writecache_error(wc, c->error, "copy error");
1720 
1721 		e = c->e;
1722 		do {
1723 			BUG_ON(!e->write_in_progress);
1724 			e->write_in_progress = false;
1725 			INIT_LIST_HEAD(&e->lru);
1726 			if (!writecache_has_error(wc))
1727 				writecache_free_entry(wc, e);
1728 
1729 			BUG_ON(!wc->writeback_size);
1730 			wc->writeback_size--;
1731 			e++;
1732 		} while (--c->n_entries);
1733 		mempool_free(c, &wc->copy_pool);
1734 	} while (!list_empty(list));
1735 }
1736 
1737 static int writecache_endio_thread(void *data)
1738 {
1739 	struct dm_writecache *wc = data;
1740 
1741 	while (1) {
1742 		struct list_head list;
1743 
1744 		raw_spin_lock_irq(&wc->endio_list_lock);
1745 		if (!list_empty(&wc->endio_list))
1746 			goto pop_from_list;
1747 		set_current_state(TASK_INTERRUPTIBLE);
1748 		raw_spin_unlock_irq(&wc->endio_list_lock);
1749 
1750 		if (unlikely(kthread_should_stop())) {
1751 			set_current_state(TASK_RUNNING);
1752 			break;
1753 		}
1754 
1755 		schedule();
1756 
1757 		continue;
1758 
1759 pop_from_list:
1760 		list = wc->endio_list;
1761 		list.next->prev = list.prev->next = &list;
1762 		INIT_LIST_HEAD(&wc->endio_list);
1763 		raw_spin_unlock_irq(&wc->endio_list_lock);
1764 
1765 		if (!WC_MODE_FUA(wc))
1766 			writecache_disk_flush(wc, wc->dev);
1767 
1768 		wc_lock(wc);
1769 
1770 		if (WC_MODE_PMEM(wc)) {
1771 			__writecache_endio_pmem(wc, &list);
1772 		} else {
1773 			__writecache_endio_ssd(wc, &list);
1774 			writecache_wait_for_ios(wc, READ);
1775 		}
1776 
1777 		writecache_commit_flushed(wc, false);
1778 
1779 		wc_unlock(wc);
1780 	}
1781 
1782 	return 0;
1783 }
1784 
1785 static bool wc_add_block(struct writeback_struct *wb, struct wc_entry *e)
1786 {
1787 	struct dm_writecache *wc = wb->wc;
1788 	unsigned block_size = wc->block_size;
1789 	void *address = memory_data(wc, e);
1790 
1791 	persistent_memory_flush_cache(address, block_size);
1792 
1793 	if (unlikely(bio_end_sector(&wb->bio) >= wc->data_device_sectors))
1794 		return true;
1795 
1796 	return bio_add_page(&wb->bio, persistent_memory_page(address),
1797 			    block_size, persistent_memory_page_offset(address)) != 0;
1798 }
1799 
1800 struct writeback_list {
1801 	struct list_head list;
1802 	size_t size;
1803 };
1804 
1805 static void __writeback_throttle(struct dm_writecache *wc, struct writeback_list *wbl)
1806 {
1807 	if (unlikely(wc->max_writeback_jobs)) {
1808 		if (READ_ONCE(wc->writeback_size) - wbl->size >= wc->max_writeback_jobs) {
1809 			wc_lock(wc);
1810 			while (wc->writeback_size - wbl->size >= wc->max_writeback_jobs)
1811 				writecache_wait_on_freelist(wc);
1812 			wc_unlock(wc);
1813 		}
1814 	}
1815 	cond_resched();
1816 }
1817 
1818 static void __writecache_writeback_pmem(struct dm_writecache *wc, struct writeback_list *wbl)
1819 {
1820 	struct wc_entry *e, *f;
1821 	struct bio *bio;
1822 	struct writeback_struct *wb;
1823 	unsigned max_pages;
1824 
1825 	while (wbl->size) {
1826 		wbl->size--;
1827 		e = container_of(wbl->list.prev, struct wc_entry, lru);
1828 		list_del(&e->lru);
1829 
1830 		max_pages = e->wc_list_contiguous;
1831 
1832 		bio = bio_alloc_bioset(wc->dev->bdev, max_pages, REQ_OP_WRITE,
1833 				       GFP_NOIO, &wc->bio_set);
1834 		wb = container_of(bio, struct writeback_struct, bio);
1835 		wb->wc = wc;
1836 		bio->bi_end_io = writecache_writeback_endio;
1837 		bio->bi_iter.bi_sector = read_original_sector(wc, e);
1838 		if (max_pages <= WB_LIST_INLINE ||
1839 		    unlikely(!(wb->wc_list = kmalloc_array(max_pages, sizeof(struct wc_entry *),
1840 							   GFP_NOIO | __GFP_NORETRY |
1841 							   __GFP_NOMEMALLOC | __GFP_NOWARN)))) {
1842 			wb->wc_list = wb->wc_list_inline;
1843 			max_pages = WB_LIST_INLINE;
1844 		}
1845 
1846 		BUG_ON(!wc_add_block(wb, e));
1847 
1848 		wb->wc_list[0] = e;
1849 		wb->wc_list_n = 1;
1850 
1851 		while (wbl->size && wb->wc_list_n < max_pages) {
1852 			f = container_of(wbl->list.prev, struct wc_entry, lru);
1853 			if (read_original_sector(wc, f) !=
1854 			    read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
1855 				break;
1856 			if (!wc_add_block(wb, f))
1857 				break;
1858 			wbl->size--;
1859 			list_del(&f->lru);
1860 			wb->wc_list[wb->wc_list_n++] = f;
1861 			e = f;
1862 		}
1863 		if (WC_MODE_FUA(wc))
1864 			bio->bi_opf |= REQ_FUA;
1865 		if (writecache_has_error(wc)) {
1866 			bio->bi_status = BLK_STS_IOERR;
1867 			bio_endio(bio);
1868 		} else if (unlikely(!bio_sectors(bio))) {
1869 			bio->bi_status = BLK_STS_OK;
1870 			bio_endio(bio);
1871 		} else {
1872 			submit_bio(bio);
1873 		}
1874 
1875 		__writeback_throttle(wc, wbl);
1876 	}
1877 }
1878 
1879 static void __writecache_writeback_ssd(struct dm_writecache *wc, struct writeback_list *wbl)
1880 {
1881 	struct wc_entry *e, *f;
1882 	struct dm_io_region from, to;
1883 	struct copy_struct *c;
1884 
1885 	while (wbl->size) {
1886 		unsigned n_sectors;
1887 
1888 		wbl->size--;
1889 		e = container_of(wbl->list.prev, struct wc_entry, lru);
1890 		list_del(&e->lru);
1891 
1892 		n_sectors = e->wc_list_contiguous << (wc->block_size_bits - SECTOR_SHIFT);
1893 
1894 		from.bdev = wc->ssd_dev->bdev;
1895 		from.sector = cache_sector(wc, e);
1896 		from.count = n_sectors;
1897 		to.bdev = wc->dev->bdev;
1898 		to.sector = read_original_sector(wc, e);
1899 		to.count = n_sectors;
1900 
1901 		c = mempool_alloc(&wc->copy_pool, GFP_NOIO);
1902 		c->wc = wc;
1903 		c->e = e;
1904 		c->n_entries = e->wc_list_contiguous;
1905 
1906 		while ((n_sectors -= wc->block_size >> SECTOR_SHIFT)) {
1907 			wbl->size--;
1908 			f = container_of(wbl->list.prev, struct wc_entry, lru);
1909 			BUG_ON(f != e + 1);
1910 			list_del(&f->lru);
1911 			e = f;
1912 		}
1913 
1914 		if (unlikely(to.sector + to.count > wc->data_device_sectors)) {
1915 			if (to.sector >= wc->data_device_sectors) {
1916 				writecache_copy_endio(0, 0, c);
1917 				continue;
1918 			}
1919 			from.count = to.count = wc->data_device_sectors - to.sector;
1920 		}
1921 
1922 		dm_kcopyd_copy(wc->dm_kcopyd, &from, 1, &to, 0, writecache_copy_endio, c);
1923 
1924 		__writeback_throttle(wc, wbl);
1925 	}
1926 }
1927 
1928 static void writecache_writeback(struct work_struct *work)
1929 {
1930 	struct dm_writecache *wc = container_of(work, struct dm_writecache, writeback_work);
1931 	struct blk_plug plug;
1932 	struct wc_entry *f, *g, *e = NULL;
1933 	struct rb_node *node, *next_node;
1934 	struct list_head skipped;
1935 	struct writeback_list wbl;
1936 	unsigned long n_walked;
1937 
1938 	if (!WC_MODE_PMEM(wc)) {
1939 		/* Wait for any active kcopyd work on behalf of ssd writeback */
1940 		dm_kcopyd_client_flush(wc->dm_kcopyd);
1941 	}
1942 
1943 	if (likely(wc->pause != 0)) {
1944 		while (1) {
1945 			unsigned long idle;
1946 			if (unlikely(wc->cleaner) || unlikely(wc->writeback_all) ||
1947 			    unlikely(dm_suspended(wc->ti)))
1948 				break;
1949 			idle = dm_iot_idle_time(&wc->iot);
1950 			if (idle >= wc->pause)
1951 				break;
1952 			idle = wc->pause - idle;
1953 			if (idle > HZ)
1954 				idle = HZ;
1955 			schedule_timeout_idle(idle);
1956 		}
1957 	}
1958 
1959 	wc_lock(wc);
1960 restart:
1961 	if (writecache_has_error(wc)) {
1962 		wc_unlock(wc);
1963 		return;
1964 	}
1965 
1966 	if (unlikely(wc->writeback_all)) {
1967 		if (writecache_wait_for_writeback(wc))
1968 			goto restart;
1969 	}
1970 
1971 	if (wc->overwrote_committed) {
1972 		writecache_wait_for_ios(wc, WRITE);
1973 	}
1974 
1975 	n_walked = 0;
1976 	INIT_LIST_HEAD(&skipped);
1977 	INIT_LIST_HEAD(&wbl.list);
1978 	wbl.size = 0;
1979 	while (!list_empty(&wc->lru) &&
1980 	       (wc->writeback_all ||
1981 		wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark ||
1982 		(jiffies - container_of(wc->lru.prev, struct wc_entry, lru)->age >=
1983 		 wc->max_age - wc->max_age / MAX_AGE_DIV))) {
1984 
1985 		n_walked++;
1986 		if (unlikely(n_walked > WRITEBACK_LATENCY) &&
1987 		    likely(!wc->writeback_all)) {
1988 			if (likely(!dm_suspended(wc->ti)))
1989 				queue_work(wc->writeback_wq, &wc->writeback_work);
1990 			break;
1991 		}
1992 
1993 		if (unlikely(wc->writeback_all)) {
1994 			if (unlikely(!e)) {
1995 				writecache_flush(wc);
1996 				e = container_of(rb_first(&wc->tree), struct wc_entry, rb_node);
1997 			} else
1998 				e = g;
1999 		} else
2000 			e = container_of(wc->lru.prev, struct wc_entry, lru);
2001 		BUG_ON(e->write_in_progress);
2002 		if (unlikely(!writecache_entry_is_committed(wc, e))) {
2003 			writecache_flush(wc);
2004 		}
2005 		node = rb_prev(&e->rb_node);
2006 		if (node) {
2007 			f = container_of(node, struct wc_entry, rb_node);
2008 			if (unlikely(read_original_sector(wc, f) ==
2009 				     read_original_sector(wc, e))) {
2010 				BUG_ON(!f->write_in_progress);
2011 				list_move(&e->lru, &skipped);
2012 				cond_resched();
2013 				continue;
2014 			}
2015 		}
2016 		wc->writeback_size++;
2017 		list_move(&e->lru, &wbl.list);
2018 		wbl.size++;
2019 		e->write_in_progress = true;
2020 		e->wc_list_contiguous = 1;
2021 
2022 		f = e;
2023 
2024 		while (1) {
2025 			next_node = rb_next(&f->rb_node);
2026 			if (unlikely(!next_node))
2027 				break;
2028 			g = container_of(next_node, struct wc_entry, rb_node);
2029 			if (unlikely(read_original_sector(wc, g) ==
2030 			    read_original_sector(wc, f))) {
2031 				f = g;
2032 				continue;
2033 			}
2034 			if (read_original_sector(wc, g) !=
2035 			    read_original_sector(wc, f) + (wc->block_size >> SECTOR_SHIFT))
2036 				break;
2037 			if (unlikely(g->write_in_progress))
2038 				break;
2039 			if (unlikely(!writecache_entry_is_committed(wc, g)))
2040 				break;
2041 
2042 			if (!WC_MODE_PMEM(wc)) {
2043 				if (g != f + 1)
2044 					break;
2045 			}
2046 
2047 			n_walked++;
2048 			//if (unlikely(n_walked > WRITEBACK_LATENCY) && likely(!wc->writeback_all))
2049 			//	break;
2050 
2051 			wc->writeback_size++;
2052 			list_move(&g->lru, &wbl.list);
2053 			wbl.size++;
2054 			g->write_in_progress = true;
2055 			g->wc_list_contiguous = BIO_MAX_VECS;
2056 			f = g;
2057 			e->wc_list_contiguous++;
2058 			if (unlikely(e->wc_list_contiguous == BIO_MAX_VECS)) {
2059 				if (unlikely(wc->writeback_all)) {
2060 					next_node = rb_next(&f->rb_node);
2061 					if (likely(next_node))
2062 						g = container_of(next_node, struct wc_entry, rb_node);
2063 				}
2064 				break;
2065 			}
2066 		}
2067 		cond_resched();
2068 	}
2069 
2070 	if (!list_empty(&skipped)) {
2071 		list_splice_tail(&skipped, &wc->lru);
2072 		/*
2073 		 * If we didn't do any progress, we must wait until some
2074 		 * writeback finishes to avoid burning CPU in a loop
2075 		 */
2076 		if (unlikely(!wbl.size))
2077 			writecache_wait_for_writeback(wc);
2078 	}
2079 
2080 	wc_unlock(wc);
2081 
2082 	blk_start_plug(&plug);
2083 
2084 	if (WC_MODE_PMEM(wc))
2085 		__writecache_writeback_pmem(wc, &wbl);
2086 	else
2087 		__writecache_writeback_ssd(wc, &wbl);
2088 
2089 	blk_finish_plug(&plug);
2090 
2091 	if (unlikely(wc->writeback_all)) {
2092 		wc_lock(wc);
2093 		while (writecache_wait_for_writeback(wc));
2094 		wc_unlock(wc);
2095 	}
2096 }
2097 
2098 static int calculate_memory_size(uint64_t device_size, unsigned block_size,
2099 				 size_t *n_blocks_p, size_t *n_metadata_blocks_p)
2100 {
2101 	uint64_t n_blocks, offset;
2102 	struct wc_entry e;
2103 
2104 	n_blocks = device_size;
2105 	do_div(n_blocks, block_size + sizeof(struct wc_memory_entry));
2106 
2107 	while (1) {
2108 		if (!n_blocks)
2109 			return -ENOSPC;
2110 		/* Verify the following entries[n_blocks] won't overflow */
2111 		if (n_blocks >= ((size_t)-sizeof(struct wc_memory_superblock) /
2112 				 sizeof(struct wc_memory_entry)))
2113 			return -EFBIG;
2114 		offset = offsetof(struct wc_memory_superblock, entries[n_blocks]);
2115 		offset = (offset + block_size - 1) & ~(uint64_t)(block_size - 1);
2116 		if (offset + n_blocks * block_size <= device_size)
2117 			break;
2118 		n_blocks--;
2119 	}
2120 
2121 	/* check if the bit field overflows */
2122 	e.index = n_blocks;
2123 	if (e.index != n_blocks)
2124 		return -EFBIG;
2125 
2126 	if (n_blocks_p)
2127 		*n_blocks_p = n_blocks;
2128 	if (n_metadata_blocks_p)
2129 		*n_metadata_blocks_p = offset >> __ffs(block_size);
2130 	return 0;
2131 }
2132 
2133 static int init_memory(struct dm_writecache *wc)
2134 {
2135 	size_t b;
2136 	int r;
2137 
2138 	r = calculate_memory_size(wc->memory_map_size, wc->block_size, &wc->n_blocks, NULL);
2139 	if (r)
2140 		return r;
2141 
2142 	r = writecache_alloc_entries(wc);
2143 	if (r)
2144 		return r;
2145 
2146 	for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++)
2147 		pmem_assign(sb(wc)->padding[b], cpu_to_le64(0));
2148 	pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION));
2149 	pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size));
2150 	pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks));
2151 	pmem_assign(sb(wc)->seq_count, cpu_to_le64(0));
2152 
2153 	for (b = 0; b < wc->n_blocks; b++) {
2154 		write_original_sector_seq_count(wc, &wc->entries[b], -1, -1);
2155 		cond_resched();
2156 	}
2157 
2158 	writecache_flush_all_metadata(wc);
2159 	writecache_commit_flushed(wc, false);
2160 	pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC));
2161 	writecache_flush_region(wc, &sb(wc)->magic, sizeof sb(wc)->magic);
2162 	writecache_commit_flushed(wc, false);
2163 
2164 	return 0;
2165 }
2166 
2167 static void writecache_dtr(struct dm_target *ti)
2168 {
2169 	struct dm_writecache *wc = ti->private;
2170 
2171 	if (!wc)
2172 		return;
2173 
2174 	if (wc->endio_thread)
2175 		kthread_stop(wc->endio_thread);
2176 
2177 	if (wc->flush_thread)
2178 		kthread_stop(wc->flush_thread);
2179 
2180 	bioset_exit(&wc->bio_set);
2181 
2182 	mempool_exit(&wc->copy_pool);
2183 
2184 	if (wc->writeback_wq)
2185 		destroy_workqueue(wc->writeback_wq);
2186 
2187 	if (wc->dev)
2188 		dm_put_device(ti, wc->dev);
2189 
2190 	if (wc->ssd_dev)
2191 		dm_put_device(ti, wc->ssd_dev);
2192 
2193 	vfree(wc->entries);
2194 
2195 	if (wc->memory_map) {
2196 		if (WC_MODE_PMEM(wc))
2197 			persistent_memory_release(wc);
2198 		else
2199 			vfree(wc->memory_map);
2200 	}
2201 
2202 	if (wc->dm_kcopyd)
2203 		dm_kcopyd_client_destroy(wc->dm_kcopyd);
2204 
2205 	if (wc->dm_io)
2206 		dm_io_client_destroy(wc->dm_io);
2207 
2208 	vfree(wc->dirty_bitmap);
2209 
2210 	kfree(wc);
2211 }
2212 
2213 static int writecache_ctr(struct dm_target *ti, unsigned argc, char **argv)
2214 {
2215 	struct dm_writecache *wc;
2216 	struct dm_arg_set as;
2217 	const char *string;
2218 	unsigned opt_params;
2219 	size_t offset, data_size;
2220 	int i, r;
2221 	char dummy;
2222 	int high_wm_percent = HIGH_WATERMARK;
2223 	int low_wm_percent = LOW_WATERMARK;
2224 	uint64_t x;
2225 	struct wc_memory_superblock s;
2226 
2227 	static struct dm_arg _args[] = {
2228 		{0, 18, "Invalid number of feature args"},
2229 	};
2230 
2231 	as.argc = argc;
2232 	as.argv = argv;
2233 
2234 	wc = kzalloc(sizeof(struct dm_writecache), GFP_KERNEL);
2235 	if (!wc) {
2236 		ti->error = "Cannot allocate writecache structure";
2237 		r = -ENOMEM;
2238 		goto bad;
2239 	}
2240 	ti->private = wc;
2241 	wc->ti = ti;
2242 
2243 	mutex_init(&wc->lock);
2244 	wc->max_age = MAX_AGE_UNSPECIFIED;
2245 	writecache_poison_lists(wc);
2246 	init_waitqueue_head(&wc->freelist_wait);
2247 	timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0);
2248 	timer_setup(&wc->max_age_timer, writecache_max_age_timer, 0);
2249 
2250 	for (i = 0; i < 2; i++) {
2251 		atomic_set(&wc->bio_in_progress[i], 0);
2252 		init_waitqueue_head(&wc->bio_in_progress_wait[i]);
2253 	}
2254 
2255 	wc->dm_io = dm_io_client_create();
2256 	if (IS_ERR(wc->dm_io)) {
2257 		r = PTR_ERR(wc->dm_io);
2258 		ti->error = "Unable to allocate dm-io client";
2259 		wc->dm_io = NULL;
2260 		goto bad;
2261 	}
2262 
2263 	wc->writeback_wq = alloc_workqueue("writecache-writeback", WQ_MEM_RECLAIM, 1);
2264 	if (!wc->writeback_wq) {
2265 		r = -ENOMEM;
2266 		ti->error = "Could not allocate writeback workqueue";
2267 		goto bad;
2268 	}
2269 	INIT_WORK(&wc->writeback_work, writecache_writeback);
2270 	INIT_WORK(&wc->flush_work, writecache_flush_work);
2271 
2272 	dm_iot_init(&wc->iot);
2273 
2274 	raw_spin_lock_init(&wc->endio_list_lock);
2275 	INIT_LIST_HEAD(&wc->endio_list);
2276 	wc->endio_thread = kthread_run(writecache_endio_thread, wc, "writecache_endio");
2277 	if (IS_ERR(wc->endio_thread)) {
2278 		r = PTR_ERR(wc->endio_thread);
2279 		wc->endio_thread = NULL;
2280 		ti->error = "Couldn't spawn endio thread";
2281 		goto bad;
2282 	}
2283 
2284 	/*
2285 	 * Parse the mode (pmem or ssd)
2286 	 */
2287 	string = dm_shift_arg(&as);
2288 	if (!string)
2289 		goto bad_arguments;
2290 
2291 	if (!strcasecmp(string, "s")) {
2292 		wc->pmem_mode = false;
2293 	} else if (!strcasecmp(string, "p")) {
2294 #ifdef DM_WRITECACHE_HAS_PMEM
2295 		wc->pmem_mode = true;
2296 		wc->writeback_fua = true;
2297 #else
2298 		/*
2299 		 * If the architecture doesn't support persistent memory or
2300 		 * the kernel doesn't support any DAX drivers, this driver can
2301 		 * only be used in SSD-only mode.
2302 		 */
2303 		r = -EOPNOTSUPP;
2304 		ti->error = "Persistent memory or DAX not supported on this system";
2305 		goto bad;
2306 #endif
2307 	} else {
2308 		goto bad_arguments;
2309 	}
2310 
2311 	if (WC_MODE_PMEM(wc)) {
2312 		r = bioset_init(&wc->bio_set, BIO_POOL_SIZE,
2313 				offsetof(struct writeback_struct, bio),
2314 				BIOSET_NEED_BVECS);
2315 		if (r) {
2316 			ti->error = "Could not allocate bio set";
2317 			goto bad;
2318 		}
2319 	} else {
2320 		wc->pause = PAUSE_WRITEBACK;
2321 		r = mempool_init_kmalloc_pool(&wc->copy_pool, 1, sizeof(struct copy_struct));
2322 		if (r) {
2323 			ti->error = "Could not allocate mempool";
2324 			goto bad;
2325 		}
2326 	}
2327 
2328 	/*
2329 	 * Parse the origin data device
2330 	 */
2331 	string = dm_shift_arg(&as);
2332 	if (!string)
2333 		goto bad_arguments;
2334 	r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->dev);
2335 	if (r) {
2336 		ti->error = "Origin data device lookup failed";
2337 		goto bad;
2338 	}
2339 
2340 	/*
2341 	 * Parse cache data device (be it pmem or ssd)
2342 	 */
2343 	string = dm_shift_arg(&as);
2344 	if (!string)
2345 		goto bad_arguments;
2346 
2347 	r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->ssd_dev);
2348 	if (r) {
2349 		ti->error = "Cache data device lookup failed";
2350 		goto bad;
2351 	}
2352 	wc->memory_map_size = bdev_nr_bytes(wc->ssd_dev->bdev);
2353 
2354 	/*
2355 	 * Parse the cache block size
2356 	 */
2357 	string = dm_shift_arg(&as);
2358 	if (!string)
2359 		goto bad_arguments;
2360 	if (sscanf(string, "%u%c", &wc->block_size, &dummy) != 1 ||
2361 	    wc->block_size < 512 || wc->block_size > PAGE_SIZE ||
2362 	    (wc->block_size & (wc->block_size - 1))) {
2363 		r = -EINVAL;
2364 		ti->error = "Invalid block size";
2365 		goto bad;
2366 	}
2367 	if (wc->block_size < bdev_logical_block_size(wc->dev->bdev) ||
2368 	    wc->block_size < bdev_logical_block_size(wc->ssd_dev->bdev)) {
2369 		r = -EINVAL;
2370 		ti->error = "Block size is smaller than device logical block size";
2371 		goto bad;
2372 	}
2373 	wc->block_size_bits = __ffs(wc->block_size);
2374 
2375 	wc->max_writeback_jobs = MAX_WRITEBACK_JOBS;
2376 	wc->autocommit_blocks = !WC_MODE_PMEM(wc) ? AUTOCOMMIT_BLOCKS_SSD : AUTOCOMMIT_BLOCKS_PMEM;
2377 	wc->autocommit_jiffies = msecs_to_jiffies(AUTOCOMMIT_MSEC);
2378 
2379 	/*
2380 	 * Parse optional arguments
2381 	 */
2382 	r = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
2383 	if (r)
2384 		goto bad;
2385 
2386 	while (opt_params) {
2387 		string = dm_shift_arg(&as), opt_params--;
2388 		if (!strcasecmp(string, "start_sector") && opt_params >= 1) {
2389 			unsigned long long start_sector;
2390 			string = dm_shift_arg(&as), opt_params--;
2391 			if (sscanf(string, "%llu%c", &start_sector, &dummy) != 1)
2392 				goto invalid_optional;
2393 			wc->start_sector = start_sector;
2394 			wc->start_sector_set = true;
2395 			if (wc->start_sector != start_sector ||
2396 			    wc->start_sector >= wc->memory_map_size >> SECTOR_SHIFT)
2397 				goto invalid_optional;
2398 		} else if (!strcasecmp(string, "high_watermark") && opt_params >= 1) {
2399 			string = dm_shift_arg(&as), opt_params--;
2400 			if (sscanf(string, "%d%c", &high_wm_percent, &dummy) != 1)
2401 				goto invalid_optional;
2402 			if (high_wm_percent < 0 || high_wm_percent > 100)
2403 				goto invalid_optional;
2404 			wc->high_wm_percent_value = high_wm_percent;
2405 			wc->high_wm_percent_set = true;
2406 		} else if (!strcasecmp(string, "low_watermark") && opt_params >= 1) {
2407 			string = dm_shift_arg(&as), opt_params--;
2408 			if (sscanf(string, "%d%c", &low_wm_percent, &dummy) != 1)
2409 				goto invalid_optional;
2410 			if (low_wm_percent < 0 || low_wm_percent > 100)
2411 				goto invalid_optional;
2412 			wc->low_wm_percent_value = low_wm_percent;
2413 			wc->low_wm_percent_set = true;
2414 		} else if (!strcasecmp(string, "writeback_jobs") && opt_params >= 1) {
2415 			string = dm_shift_arg(&as), opt_params--;
2416 			if (sscanf(string, "%u%c", &wc->max_writeback_jobs, &dummy) != 1)
2417 				goto invalid_optional;
2418 			wc->max_writeback_jobs_set = true;
2419 		} else if (!strcasecmp(string, "autocommit_blocks") && opt_params >= 1) {
2420 			string = dm_shift_arg(&as), opt_params--;
2421 			if (sscanf(string, "%u%c", &wc->autocommit_blocks, &dummy) != 1)
2422 				goto invalid_optional;
2423 			wc->autocommit_blocks_set = true;
2424 		} else if (!strcasecmp(string, "autocommit_time") && opt_params >= 1) {
2425 			unsigned autocommit_msecs;
2426 			string = dm_shift_arg(&as), opt_params--;
2427 			if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1)
2428 				goto invalid_optional;
2429 			if (autocommit_msecs > 3600000)
2430 				goto invalid_optional;
2431 			wc->autocommit_jiffies = msecs_to_jiffies(autocommit_msecs);
2432 			wc->autocommit_time_value = autocommit_msecs;
2433 			wc->autocommit_time_set = true;
2434 		} else if (!strcasecmp(string, "max_age") && opt_params >= 1) {
2435 			unsigned max_age_msecs;
2436 			string = dm_shift_arg(&as), opt_params--;
2437 			if (sscanf(string, "%u%c", &max_age_msecs, &dummy) != 1)
2438 				goto invalid_optional;
2439 			if (max_age_msecs > 86400000)
2440 				goto invalid_optional;
2441 			wc->max_age = msecs_to_jiffies(max_age_msecs);
2442 			wc->max_age_set = true;
2443 			wc->max_age_value = max_age_msecs;
2444 		} else if (!strcasecmp(string, "cleaner")) {
2445 			wc->cleaner_set = true;
2446 			wc->cleaner = true;
2447 		} else if (!strcasecmp(string, "fua")) {
2448 			if (WC_MODE_PMEM(wc)) {
2449 				wc->writeback_fua = true;
2450 				wc->writeback_fua_set = true;
2451 			} else goto invalid_optional;
2452 		} else if (!strcasecmp(string, "nofua")) {
2453 			if (WC_MODE_PMEM(wc)) {
2454 				wc->writeback_fua = false;
2455 				wc->writeback_fua_set = true;
2456 			} else goto invalid_optional;
2457 		} else if (!strcasecmp(string, "metadata_only")) {
2458 			wc->metadata_only = true;
2459 		} else if (!strcasecmp(string, "pause_writeback") && opt_params >= 1) {
2460 			unsigned pause_msecs;
2461 			if (WC_MODE_PMEM(wc))
2462 				goto invalid_optional;
2463 			string = dm_shift_arg(&as), opt_params--;
2464 			if (sscanf(string, "%u%c", &pause_msecs, &dummy) != 1)
2465 				goto invalid_optional;
2466 			if (pause_msecs > 60000)
2467 				goto invalid_optional;
2468 			wc->pause = msecs_to_jiffies(pause_msecs);
2469 			wc->pause_set = true;
2470 			wc->pause_value = pause_msecs;
2471 		} else {
2472 invalid_optional:
2473 			r = -EINVAL;
2474 			ti->error = "Invalid optional argument";
2475 			goto bad;
2476 		}
2477 	}
2478 
2479 	if (high_wm_percent < low_wm_percent) {
2480 		r = -EINVAL;
2481 		ti->error = "High watermark must be greater than or equal to low watermark";
2482 		goto bad;
2483 	}
2484 
2485 	if (WC_MODE_PMEM(wc)) {
2486 		if (!dax_synchronous(wc->ssd_dev->dax_dev)) {
2487 			r = -EOPNOTSUPP;
2488 			ti->error = "Asynchronous persistent memory not supported as pmem cache";
2489 			goto bad;
2490 		}
2491 
2492 		r = persistent_memory_claim(wc);
2493 		if (r) {
2494 			ti->error = "Unable to map persistent memory for cache";
2495 			goto bad;
2496 		}
2497 	} else {
2498 		size_t n_blocks, n_metadata_blocks;
2499 		uint64_t n_bitmap_bits;
2500 
2501 		wc->memory_map_size -= (uint64_t)wc->start_sector << SECTOR_SHIFT;
2502 
2503 		bio_list_init(&wc->flush_list);
2504 		wc->flush_thread = kthread_run(writecache_flush_thread, wc, "dm_writecache_flush");
2505 		if (IS_ERR(wc->flush_thread)) {
2506 			r = PTR_ERR(wc->flush_thread);
2507 			wc->flush_thread = NULL;
2508 			ti->error = "Couldn't spawn flush thread";
2509 			goto bad;
2510 		}
2511 
2512 		r = calculate_memory_size(wc->memory_map_size, wc->block_size,
2513 					  &n_blocks, &n_metadata_blocks);
2514 		if (r) {
2515 			ti->error = "Invalid device size";
2516 			goto bad;
2517 		}
2518 
2519 		n_bitmap_bits = (((uint64_t)n_metadata_blocks << wc->block_size_bits) +
2520 				 BITMAP_GRANULARITY - 1) / BITMAP_GRANULARITY;
2521 		/* this is limitation of test_bit functions */
2522 		if (n_bitmap_bits > 1U << 31) {
2523 			r = -EFBIG;
2524 			ti->error = "Invalid device size";
2525 			goto bad;
2526 		}
2527 
2528 		wc->memory_map = vmalloc(n_metadata_blocks << wc->block_size_bits);
2529 		if (!wc->memory_map) {
2530 			r = -ENOMEM;
2531 			ti->error = "Unable to allocate memory for metadata";
2532 			goto bad;
2533 		}
2534 
2535 		wc->dm_kcopyd = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2536 		if (IS_ERR(wc->dm_kcopyd)) {
2537 			r = PTR_ERR(wc->dm_kcopyd);
2538 			ti->error = "Unable to allocate dm-kcopyd client";
2539 			wc->dm_kcopyd = NULL;
2540 			goto bad;
2541 		}
2542 
2543 		wc->metadata_sectors = n_metadata_blocks << (wc->block_size_bits - SECTOR_SHIFT);
2544 		wc->dirty_bitmap_size = (n_bitmap_bits + BITS_PER_LONG - 1) /
2545 			BITS_PER_LONG * sizeof(unsigned long);
2546 		wc->dirty_bitmap = vzalloc(wc->dirty_bitmap_size);
2547 		if (!wc->dirty_bitmap) {
2548 			r = -ENOMEM;
2549 			ti->error = "Unable to allocate dirty bitmap";
2550 			goto bad;
2551 		}
2552 
2553 		r = writecache_read_metadata(wc, wc->block_size >> SECTOR_SHIFT);
2554 		if (r) {
2555 			ti->error = "Unable to read first block of metadata";
2556 			goto bad;
2557 		}
2558 	}
2559 
2560 	r = copy_mc_to_kernel(&s, sb(wc), sizeof(struct wc_memory_superblock));
2561 	if (r) {
2562 		ti->error = "Hardware memory error when reading superblock";
2563 		goto bad;
2564 	}
2565 	if (!le32_to_cpu(s.magic) && !le32_to_cpu(s.version)) {
2566 		r = init_memory(wc);
2567 		if (r) {
2568 			ti->error = "Unable to initialize device";
2569 			goto bad;
2570 		}
2571 		r = copy_mc_to_kernel(&s, sb(wc),
2572 				      sizeof(struct wc_memory_superblock));
2573 		if (r) {
2574 			ti->error = "Hardware memory error when reading superblock";
2575 			goto bad;
2576 		}
2577 	}
2578 
2579 	if (le32_to_cpu(s.magic) != MEMORY_SUPERBLOCK_MAGIC) {
2580 		ti->error = "Invalid magic in the superblock";
2581 		r = -EINVAL;
2582 		goto bad;
2583 	}
2584 
2585 	if (le32_to_cpu(s.version) != MEMORY_SUPERBLOCK_VERSION) {
2586 		ti->error = "Invalid version in the superblock";
2587 		r = -EINVAL;
2588 		goto bad;
2589 	}
2590 
2591 	if (le32_to_cpu(s.block_size) != wc->block_size) {
2592 		ti->error = "Block size does not match superblock";
2593 		r = -EINVAL;
2594 		goto bad;
2595 	}
2596 
2597 	wc->n_blocks = le64_to_cpu(s.n_blocks);
2598 
2599 	offset = wc->n_blocks * sizeof(struct wc_memory_entry);
2600 	if (offset / sizeof(struct wc_memory_entry) != le64_to_cpu(sb(wc)->n_blocks)) {
2601 overflow:
2602 		ti->error = "Overflow in size calculation";
2603 		r = -EINVAL;
2604 		goto bad;
2605 	}
2606 	offset += sizeof(struct wc_memory_superblock);
2607 	if (offset < sizeof(struct wc_memory_superblock))
2608 		goto overflow;
2609 	offset = (offset + wc->block_size - 1) & ~(size_t)(wc->block_size - 1);
2610 	data_size = wc->n_blocks * (size_t)wc->block_size;
2611 	if (!offset || (data_size / wc->block_size != wc->n_blocks) ||
2612 	    (offset + data_size < offset))
2613 		goto overflow;
2614 	if (offset + data_size > wc->memory_map_size) {
2615 		ti->error = "Memory area is too small";
2616 		r = -EINVAL;
2617 		goto bad;
2618 	}
2619 
2620 	wc->metadata_sectors = offset >> SECTOR_SHIFT;
2621 	wc->block_start = (char *)sb(wc) + offset;
2622 
2623 	x = (uint64_t)wc->n_blocks * (100 - high_wm_percent);
2624 	x += 50;
2625 	do_div(x, 100);
2626 	wc->freelist_high_watermark = x;
2627 	x = (uint64_t)wc->n_blocks * (100 - low_wm_percent);
2628 	x += 50;
2629 	do_div(x, 100);
2630 	wc->freelist_low_watermark = x;
2631 
2632 	if (wc->cleaner)
2633 		activate_cleaner(wc);
2634 
2635 	r = writecache_alloc_entries(wc);
2636 	if (r) {
2637 		ti->error = "Cannot allocate memory";
2638 		goto bad;
2639 	}
2640 
2641 	ti->num_flush_bios = WC_MODE_PMEM(wc) ? 1 : 2;
2642 	ti->flush_supported = true;
2643 	ti->num_discard_bios = 1;
2644 
2645 	if (WC_MODE_PMEM(wc))
2646 		persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
2647 
2648 	return 0;
2649 
2650 bad_arguments:
2651 	r = -EINVAL;
2652 	ti->error = "Bad arguments";
2653 bad:
2654 	writecache_dtr(ti);
2655 	return r;
2656 }
2657 
2658 static void writecache_status(struct dm_target *ti, status_type_t type,
2659 			      unsigned status_flags, char *result, unsigned maxlen)
2660 {
2661 	struct dm_writecache *wc = ti->private;
2662 	unsigned extra_args;
2663 	unsigned sz = 0;
2664 
2665 	switch (type) {
2666 	case STATUSTYPE_INFO:
2667 		DMEMIT("%ld %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu",
2668 		       writecache_has_error(wc),
2669 		       (unsigned long long)wc->n_blocks, (unsigned long long)wc->freelist_size,
2670 		       (unsigned long long)wc->writeback_size,
2671 		       wc->stats.reads,
2672 		       wc->stats.read_hits,
2673 		       wc->stats.writes,
2674 		       wc->stats.write_hits_uncommitted,
2675 		       wc->stats.write_hits_committed,
2676 		       wc->stats.writes_around,
2677 		       wc->stats.writes_allocate,
2678 		       wc->stats.writes_blocked_on_freelist,
2679 		       wc->stats.flushes,
2680 		       wc->stats.discards);
2681 		break;
2682 	case STATUSTYPE_TABLE:
2683 		DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's',
2684 				wc->dev->name, wc->ssd_dev->name, wc->block_size);
2685 		extra_args = 0;
2686 		if (wc->start_sector_set)
2687 			extra_args += 2;
2688 		if (wc->high_wm_percent_set)
2689 			extra_args += 2;
2690 		if (wc->low_wm_percent_set)
2691 			extra_args += 2;
2692 		if (wc->max_writeback_jobs_set)
2693 			extra_args += 2;
2694 		if (wc->autocommit_blocks_set)
2695 			extra_args += 2;
2696 		if (wc->autocommit_time_set)
2697 			extra_args += 2;
2698 		if (wc->max_age_set)
2699 			extra_args += 2;
2700 		if (wc->cleaner_set)
2701 			extra_args++;
2702 		if (wc->writeback_fua_set)
2703 			extra_args++;
2704 		if (wc->metadata_only)
2705 			extra_args++;
2706 		if (wc->pause_set)
2707 			extra_args += 2;
2708 
2709 		DMEMIT("%u", extra_args);
2710 		if (wc->start_sector_set)
2711 			DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector);
2712 		if (wc->high_wm_percent_set)
2713 			DMEMIT(" high_watermark %u", wc->high_wm_percent_value);
2714 		if (wc->low_wm_percent_set)
2715 			DMEMIT(" low_watermark %u", wc->low_wm_percent_value);
2716 		if (wc->max_writeback_jobs_set)
2717 			DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs);
2718 		if (wc->autocommit_blocks_set)
2719 			DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks);
2720 		if (wc->autocommit_time_set)
2721 			DMEMIT(" autocommit_time %u", wc->autocommit_time_value);
2722 		if (wc->max_age_set)
2723 			DMEMIT(" max_age %u", wc->max_age_value);
2724 		if (wc->cleaner_set)
2725 			DMEMIT(" cleaner");
2726 		if (wc->writeback_fua_set)
2727 			DMEMIT(" %sfua", wc->writeback_fua ? "" : "no");
2728 		if (wc->metadata_only)
2729 			DMEMIT(" metadata_only");
2730 		if (wc->pause_set)
2731 			DMEMIT(" pause_writeback %u", wc->pause_value);
2732 		break;
2733 	case STATUSTYPE_IMA:
2734 		*result = '\0';
2735 		break;
2736 	}
2737 }
2738 
2739 static struct target_type writecache_target = {
2740 	.name			= "writecache",
2741 	.version		= {1, 6, 0},
2742 	.module			= THIS_MODULE,
2743 	.ctr			= writecache_ctr,
2744 	.dtr			= writecache_dtr,
2745 	.status			= writecache_status,
2746 	.postsuspend		= writecache_suspend,
2747 	.resume			= writecache_resume,
2748 	.message		= writecache_message,
2749 	.map			= writecache_map,
2750 	.end_io			= writecache_end_io,
2751 	.iterate_devices	= writecache_iterate_devices,
2752 	.io_hints		= writecache_io_hints,
2753 };
2754 
2755 static int __init dm_writecache_init(void)
2756 {
2757 	int r;
2758 
2759 	r = dm_register_target(&writecache_target);
2760 	if (r < 0) {
2761 		DMERR("register failed %d", r);
2762 		return r;
2763 	}
2764 
2765 	return 0;
2766 }
2767 
2768 static void __exit dm_writecache_exit(void)
2769 {
2770 	dm_unregister_target(&writecache_target);
2771 }
2772 
2773 module_init(dm_writecache_init);
2774 module_exit(dm_writecache_exit);
2775 
2776 MODULE_DESCRIPTION(DM_NAME " writecache target");
2777 MODULE_AUTHOR("Mikulas Patocka <[email protected]>");
2778 MODULE_LICENSE("GPL");
2779