1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2005-2007 Red Hat GmbH 4 * 5 * A target that delays reads and/or writes and can send 6 * them to different devices. 7 * 8 * This file is released under the GPL. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/init.h> 13 #include <linux/blkdev.h> 14 #include <linux/bio.h> 15 #include <linux/slab.h> 16 #include <linux/kthread.h> 17 18 #include <linux/device-mapper.h> 19 20 #define DM_MSG_PREFIX "delay" 21 22 struct delay_class { 23 struct dm_dev *dev; 24 sector_t start; 25 unsigned int delay; 26 unsigned int ops; 27 }; 28 29 struct delay_c { 30 struct timer_list delay_timer; 31 struct mutex timer_lock; 32 struct mutex process_bios_lock; /* hold while removing bios to be processed from list */ 33 spinlock_t delayed_bios_lock; /* hold on all accesses to delayed_bios list */ 34 struct workqueue_struct *kdelayd_wq; 35 struct work_struct flush_expired_bios; 36 struct list_head delayed_bios; 37 struct task_struct *worker; 38 bool may_delay; 39 40 struct delay_class read; 41 struct delay_class write; 42 struct delay_class flush; 43 44 int argc; 45 }; 46 47 struct dm_delay_info { 48 struct delay_c *context; 49 struct delay_class *class; 50 struct list_head list; 51 unsigned long expires; 52 }; 53 54 static void handle_delayed_timer(struct timer_list *t) 55 { 56 struct delay_c *dc = from_timer(dc, t, delay_timer); 57 58 queue_work(dc->kdelayd_wq, &dc->flush_expired_bios); 59 } 60 61 static void queue_timeout(struct delay_c *dc, unsigned long expires) 62 { 63 mutex_lock(&dc->timer_lock); 64 65 if (!timer_pending(&dc->delay_timer) || expires < dc->delay_timer.expires) 66 mod_timer(&dc->delay_timer, expires); 67 68 mutex_unlock(&dc->timer_lock); 69 } 70 71 static inline bool delay_is_fast(struct delay_c *dc) 72 { 73 return !!dc->worker; 74 } 75 76 static void flush_bios(struct bio *bio) 77 { 78 struct bio *n; 79 80 while (bio) { 81 n = bio->bi_next; 82 bio->bi_next = NULL; 83 dm_submit_bio_remap(bio, NULL); 84 bio = n; 85 } 86 } 87 88 static void flush_delayed_bios(struct delay_c *dc, bool flush_all) 89 { 90 struct dm_delay_info *delayed, *next; 91 struct bio_list flush_bio_list; 92 LIST_HEAD(local_list); 93 unsigned long next_expires = 0; 94 bool start_timer = false; 95 bio_list_init(&flush_bio_list); 96 97 mutex_lock(&dc->process_bios_lock); 98 spin_lock(&dc->delayed_bios_lock); 99 list_replace_init(&dc->delayed_bios, &local_list); 100 spin_unlock(&dc->delayed_bios_lock); 101 list_for_each_entry_safe(delayed, next, &local_list, list) { 102 cond_resched(); 103 if (flush_all || time_after_eq(jiffies, delayed->expires)) { 104 struct bio *bio = dm_bio_from_per_bio_data(delayed, 105 sizeof(struct dm_delay_info)); 106 list_del(&delayed->list); 107 bio_list_add(&flush_bio_list, bio); 108 delayed->class->ops--; 109 continue; 110 } 111 112 if (!delay_is_fast(dc)) { 113 if (!start_timer) { 114 start_timer = true; 115 next_expires = delayed->expires; 116 } else { 117 next_expires = min(next_expires, delayed->expires); 118 } 119 } 120 } 121 spin_lock(&dc->delayed_bios_lock); 122 list_splice(&local_list, &dc->delayed_bios); 123 spin_unlock(&dc->delayed_bios_lock); 124 mutex_unlock(&dc->process_bios_lock); 125 126 if (start_timer) 127 queue_timeout(dc, next_expires); 128 129 flush_bios(bio_list_get(&flush_bio_list)); 130 } 131 132 static int flush_worker_fn(void *data) 133 { 134 struct delay_c *dc = data; 135 136 while (!kthread_should_stop()) { 137 flush_delayed_bios(dc, false); 138 spin_lock(&dc->delayed_bios_lock); 139 if (unlikely(list_empty(&dc->delayed_bios))) { 140 set_current_state(TASK_INTERRUPTIBLE); 141 spin_unlock(&dc->delayed_bios_lock); 142 schedule(); 143 } else { 144 spin_unlock(&dc->delayed_bios_lock); 145 cond_resched(); 146 } 147 } 148 149 return 0; 150 } 151 152 static void flush_expired_bios(struct work_struct *work) 153 { 154 struct delay_c *dc; 155 156 dc = container_of(work, struct delay_c, flush_expired_bios); 157 flush_delayed_bios(dc, false); 158 } 159 160 static void delay_dtr(struct dm_target *ti) 161 { 162 struct delay_c *dc = ti->private; 163 164 if (dc->kdelayd_wq) { 165 timer_shutdown_sync(&dc->delay_timer); 166 destroy_workqueue(dc->kdelayd_wq); 167 } 168 169 if (dc->read.dev) 170 dm_put_device(ti, dc->read.dev); 171 if (dc->write.dev) 172 dm_put_device(ti, dc->write.dev); 173 if (dc->flush.dev) 174 dm_put_device(ti, dc->flush.dev); 175 if (dc->worker) 176 kthread_stop(dc->worker); 177 178 mutex_destroy(&dc->process_bios_lock); 179 mutex_destroy(&dc->timer_lock); 180 181 kfree(dc); 182 } 183 184 static int delay_class_ctr(struct dm_target *ti, struct delay_class *c, char **argv) 185 { 186 int ret; 187 unsigned long long tmpll; 188 char dummy; 189 190 if (sscanf(argv[1], "%llu%c", &tmpll, &dummy) != 1 || tmpll != (sector_t)tmpll) { 191 ti->error = "Invalid device sector"; 192 return -EINVAL; 193 } 194 c->start = tmpll; 195 196 if (sscanf(argv[2], "%u%c", &c->delay, &dummy) != 1) { 197 ti->error = "Invalid delay"; 198 return -EINVAL; 199 } 200 201 ret = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &c->dev); 202 if (ret) { 203 ti->error = "Device lookup failed"; 204 return ret; 205 } 206 207 return 0; 208 } 209 210 /* 211 * Mapping parameters: 212 * <device> <offset> <delay> [<write_device> <write_offset> <write_delay>] 213 * 214 * With separate write parameters, the first set is only used for reads. 215 * Offsets are specified in sectors. 216 * Delays are specified in milliseconds. 217 */ 218 static int delay_ctr(struct dm_target *ti, unsigned int argc, char **argv) 219 { 220 struct delay_c *dc; 221 int ret; 222 unsigned int max_delay; 223 224 if (argc != 3 && argc != 6 && argc != 9) { 225 ti->error = "Requires exactly 3, 6 or 9 arguments"; 226 return -EINVAL; 227 } 228 229 dc = kzalloc(sizeof(*dc), GFP_KERNEL); 230 if (!dc) { 231 ti->error = "Cannot allocate context"; 232 return -ENOMEM; 233 } 234 235 ti->private = dc; 236 INIT_LIST_HEAD(&dc->delayed_bios); 237 mutex_init(&dc->timer_lock); 238 mutex_init(&dc->process_bios_lock); 239 spin_lock_init(&dc->delayed_bios_lock); 240 dc->may_delay = true; 241 dc->argc = argc; 242 243 ret = delay_class_ctr(ti, &dc->read, argv); 244 if (ret) 245 goto bad; 246 max_delay = dc->read.delay; 247 248 if (argc == 3) { 249 ret = delay_class_ctr(ti, &dc->write, argv); 250 if (ret) 251 goto bad; 252 ret = delay_class_ctr(ti, &dc->flush, argv); 253 if (ret) 254 goto bad; 255 goto out; 256 } 257 258 ret = delay_class_ctr(ti, &dc->write, argv + 3); 259 if (ret) 260 goto bad; 261 max_delay = max(max_delay, dc->write.delay); 262 263 if (argc == 6) { 264 ret = delay_class_ctr(ti, &dc->flush, argv + 3); 265 if (ret) 266 goto bad; 267 goto out; 268 } 269 270 ret = delay_class_ctr(ti, &dc->flush, argv + 6); 271 if (ret) 272 goto bad; 273 max_delay = max(max_delay, dc->flush.delay); 274 275 out: 276 if (max_delay < 50) { 277 /* 278 * In case of small requested delays, use kthread instead of 279 * timers and workqueue to achieve better latency. 280 */ 281 dc->worker = kthread_run(&flush_worker_fn, dc, "dm-delay-flush-worker"); 282 if (IS_ERR(dc->worker)) { 283 ret = PTR_ERR(dc->worker); 284 dc->worker = NULL; 285 goto bad; 286 } 287 } else { 288 timer_setup(&dc->delay_timer, handle_delayed_timer, 0); 289 INIT_WORK(&dc->flush_expired_bios, flush_expired_bios); 290 dc->kdelayd_wq = alloc_workqueue("kdelayd", WQ_MEM_RECLAIM, 0); 291 if (!dc->kdelayd_wq) { 292 ret = -EINVAL; 293 DMERR("Couldn't start kdelayd"); 294 goto bad; 295 } 296 } 297 298 ti->num_flush_bios = 1; 299 ti->num_discard_bios = 1; 300 ti->accounts_remapped_io = true; 301 ti->per_io_data_size = sizeof(struct dm_delay_info); 302 return 0; 303 304 bad: 305 delay_dtr(ti); 306 return ret; 307 } 308 309 static int delay_bio(struct delay_c *dc, struct delay_class *c, struct bio *bio) 310 { 311 struct dm_delay_info *delayed; 312 unsigned long expires = 0; 313 314 if (!c->delay) 315 return DM_MAPIO_REMAPPED; 316 317 delayed = dm_per_bio_data(bio, sizeof(struct dm_delay_info)); 318 319 delayed->context = dc; 320 delayed->expires = expires = jiffies + msecs_to_jiffies(c->delay); 321 322 spin_lock(&dc->delayed_bios_lock); 323 if (unlikely(!dc->may_delay)) { 324 spin_unlock(&dc->delayed_bios_lock); 325 return DM_MAPIO_REMAPPED; 326 } 327 c->ops++; 328 list_add_tail(&delayed->list, &dc->delayed_bios); 329 spin_unlock(&dc->delayed_bios_lock); 330 331 if (delay_is_fast(dc)) 332 wake_up_process(dc->worker); 333 else 334 queue_timeout(dc, expires); 335 336 return DM_MAPIO_SUBMITTED; 337 } 338 339 static void delay_presuspend(struct dm_target *ti) 340 { 341 struct delay_c *dc = ti->private; 342 343 spin_lock(&dc->delayed_bios_lock); 344 dc->may_delay = false; 345 spin_unlock(&dc->delayed_bios_lock); 346 347 if (!delay_is_fast(dc)) 348 timer_delete(&dc->delay_timer); 349 flush_delayed_bios(dc, true); 350 } 351 352 static void delay_resume(struct dm_target *ti) 353 { 354 struct delay_c *dc = ti->private; 355 356 dc->may_delay = true; 357 } 358 359 static int delay_map(struct dm_target *ti, struct bio *bio) 360 { 361 struct delay_c *dc = ti->private; 362 struct delay_class *c; 363 struct dm_delay_info *delayed = dm_per_bio_data(bio, sizeof(struct dm_delay_info)); 364 365 if (bio_data_dir(bio) == WRITE) { 366 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) 367 c = &dc->flush; 368 else 369 c = &dc->write; 370 } else { 371 c = &dc->read; 372 } 373 delayed->class = c; 374 bio_set_dev(bio, c->dev->bdev); 375 bio->bi_iter.bi_sector = c->start + dm_target_offset(ti, bio->bi_iter.bi_sector); 376 377 return delay_bio(dc, c, bio); 378 } 379 380 #define DMEMIT_DELAY_CLASS(c) \ 381 DMEMIT("%s %llu %u", (c)->dev->name, (unsigned long long)(c)->start, (c)->delay) 382 383 static void delay_status(struct dm_target *ti, status_type_t type, 384 unsigned int status_flags, char *result, unsigned int maxlen) 385 { 386 struct delay_c *dc = ti->private; 387 int sz = 0; 388 389 switch (type) { 390 case STATUSTYPE_INFO: 391 DMEMIT("%u %u %u", dc->read.ops, dc->write.ops, dc->flush.ops); 392 break; 393 394 case STATUSTYPE_TABLE: 395 DMEMIT_DELAY_CLASS(&dc->read); 396 if (dc->argc >= 6) { 397 DMEMIT(" "); 398 DMEMIT_DELAY_CLASS(&dc->write); 399 } 400 if (dc->argc >= 9) { 401 DMEMIT(" "); 402 DMEMIT_DELAY_CLASS(&dc->flush); 403 } 404 break; 405 406 case STATUSTYPE_IMA: 407 *result = '\0'; 408 break; 409 } 410 } 411 412 static int delay_iterate_devices(struct dm_target *ti, 413 iterate_devices_callout_fn fn, void *data) 414 { 415 struct delay_c *dc = ti->private; 416 int ret = 0; 417 418 ret = fn(ti, dc->read.dev, dc->read.start, ti->len, data); 419 if (ret) 420 goto out; 421 ret = fn(ti, dc->write.dev, dc->write.start, ti->len, data); 422 if (ret) 423 goto out; 424 ret = fn(ti, dc->flush.dev, dc->flush.start, ti->len, data); 425 if (ret) 426 goto out; 427 428 out: 429 return ret; 430 } 431 432 static struct target_type delay_target = { 433 .name = "delay", 434 .version = {1, 4, 0}, 435 .features = DM_TARGET_PASSES_INTEGRITY, 436 .module = THIS_MODULE, 437 .ctr = delay_ctr, 438 .dtr = delay_dtr, 439 .map = delay_map, 440 .presuspend = delay_presuspend, 441 .resume = delay_resume, 442 .status = delay_status, 443 .iterate_devices = delay_iterate_devices, 444 }; 445 module_dm(delay); 446 447 MODULE_DESCRIPTION(DM_NAME " delay target"); 448 MODULE_AUTHOR("Heinz Mauelshagen <[email protected]>"); 449 MODULE_LICENSE("GPL"); 450