1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/io_uring.h> 6 7 #include <trace/events/io_uring.h> 8 9 #include <uapi/linux/io_uring.h> 10 11 #include "io_uring.h" 12 #include "refs.h" 13 #include "cancel.h" 14 #include "timeout.h" 15 16 struct io_timeout { 17 struct file *file; 18 u32 off; 19 u32 target_seq; 20 struct list_head list; 21 /* head of the link, used by linked timeouts only */ 22 struct io_kiocb *head; 23 /* for linked completions */ 24 struct io_kiocb *prev; 25 }; 26 27 struct io_timeout_rem { 28 struct file *file; 29 u64 addr; 30 31 /* timeout update */ 32 struct timespec64 ts; 33 u32 flags; 34 bool ltimeout; 35 }; 36 37 static inline bool io_is_timeout_noseq(struct io_kiocb *req) 38 { 39 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 40 41 return !timeout->off; 42 } 43 44 static inline void io_put_req(struct io_kiocb *req) 45 { 46 if (req_ref_put_and_test(req)) { 47 io_queue_next(req); 48 io_free_req(req); 49 } 50 } 51 52 static bool io_kill_timeout(struct io_kiocb *req, int status) 53 __must_hold(&req->ctx->completion_lock) 54 __must_hold(&req->ctx->timeout_lock) 55 { 56 struct io_timeout_data *io = req->async_data; 57 58 if (hrtimer_try_to_cancel(&io->timer) != -1) { 59 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 60 61 if (status) 62 req_set_fail(req); 63 atomic_set(&req->ctx->cq_timeouts, 64 atomic_read(&req->ctx->cq_timeouts) + 1); 65 list_del_init(&timeout->list); 66 io_req_queue_tw_complete(req, status); 67 return true; 68 } 69 return false; 70 } 71 72 __cold void io_flush_timeouts(struct io_ring_ctx *ctx) 73 __must_hold(&ctx->completion_lock) 74 { 75 u32 seq; 76 struct io_timeout *timeout, *tmp; 77 78 spin_lock_irq(&ctx->timeout_lock); 79 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); 80 81 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 82 struct io_kiocb *req = cmd_to_io_kiocb(timeout); 83 u32 events_needed, events_got; 84 85 if (io_is_timeout_noseq(req)) 86 break; 87 88 /* 89 * Since seq can easily wrap around over time, subtract 90 * the last seq at which timeouts were flushed before comparing. 91 * Assuming not more than 2^31-1 events have happened since, 92 * these subtractions won't have wrapped, so we can check if 93 * target is in [last_seq, current_seq] by comparing the two. 94 */ 95 events_needed = timeout->target_seq - ctx->cq_last_tm_flush; 96 events_got = seq - ctx->cq_last_tm_flush; 97 if (events_got < events_needed) 98 break; 99 100 io_kill_timeout(req, 0); 101 } 102 ctx->cq_last_tm_flush = seq; 103 spin_unlock_irq(&ctx->timeout_lock); 104 } 105 106 static void io_req_tw_fail_links(struct io_kiocb *link, bool *locked) 107 { 108 io_tw_lock(link->ctx, locked); 109 while (link) { 110 struct io_kiocb *nxt = link->link; 111 long res = -ECANCELED; 112 113 if (link->flags & REQ_F_FAIL) 114 res = link->cqe.res; 115 link->link = NULL; 116 io_req_set_res(link, res, 0); 117 io_req_task_complete(link, locked); 118 link = nxt; 119 } 120 } 121 122 static void io_fail_links(struct io_kiocb *req) 123 __must_hold(&req->ctx->completion_lock) 124 { 125 struct io_kiocb *link = req->link; 126 bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES; 127 128 if (!link) 129 return; 130 131 while (link) { 132 if (ignore_cqes) 133 link->flags |= REQ_F_CQE_SKIP; 134 else 135 link->flags &= ~REQ_F_CQE_SKIP; 136 trace_io_uring_fail_link(req, link); 137 link = link->link; 138 } 139 140 link = req->link; 141 link->io_task_work.func = io_req_tw_fail_links; 142 io_req_task_work_add(link); 143 req->link = NULL; 144 } 145 146 static inline void io_remove_next_linked(struct io_kiocb *req) 147 { 148 struct io_kiocb *nxt = req->link; 149 150 req->link = nxt->link; 151 nxt->link = NULL; 152 } 153 154 void io_disarm_next(struct io_kiocb *req) 155 __must_hold(&req->ctx->completion_lock) 156 { 157 struct io_kiocb *link = NULL; 158 159 if (req->flags & REQ_F_ARM_LTIMEOUT) { 160 link = req->link; 161 req->flags &= ~REQ_F_ARM_LTIMEOUT; 162 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { 163 io_remove_next_linked(req); 164 io_req_queue_tw_complete(link, -ECANCELED); 165 } 166 } else if (req->flags & REQ_F_LINK_TIMEOUT) { 167 struct io_ring_ctx *ctx = req->ctx; 168 169 spin_lock_irq(&ctx->timeout_lock); 170 link = io_disarm_linked_timeout(req); 171 spin_unlock_irq(&ctx->timeout_lock); 172 if (link) 173 io_req_queue_tw_complete(link, -ECANCELED); 174 } 175 if (unlikely((req->flags & REQ_F_FAIL) && 176 !(req->flags & REQ_F_HARDLINK))) 177 io_fail_links(req); 178 } 179 180 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, 181 struct io_kiocb *link) 182 __must_hold(&req->ctx->completion_lock) 183 __must_hold(&req->ctx->timeout_lock) 184 { 185 struct io_timeout_data *io = link->async_data; 186 struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout); 187 188 io_remove_next_linked(req); 189 timeout->head = NULL; 190 if (hrtimer_try_to_cancel(&io->timer) != -1) { 191 list_del(&timeout->list); 192 return link; 193 } 194 195 return NULL; 196 } 197 198 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) 199 { 200 struct io_timeout_data *data = container_of(timer, 201 struct io_timeout_data, timer); 202 struct io_kiocb *req = data->req; 203 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 204 struct io_ring_ctx *ctx = req->ctx; 205 unsigned long flags; 206 207 spin_lock_irqsave(&ctx->timeout_lock, flags); 208 list_del_init(&timeout->list); 209 atomic_set(&req->ctx->cq_timeouts, 210 atomic_read(&req->ctx->cq_timeouts) + 1); 211 spin_unlock_irqrestore(&ctx->timeout_lock, flags); 212 213 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS)) 214 req_set_fail(req); 215 216 io_req_set_res(req, -ETIME, 0); 217 req->io_task_work.func = io_req_task_complete; 218 io_req_task_work_add(req); 219 return HRTIMER_NORESTART; 220 } 221 222 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, 223 struct io_cancel_data *cd) 224 __must_hold(&ctx->timeout_lock) 225 { 226 struct io_timeout *timeout; 227 struct io_timeout_data *io; 228 struct io_kiocb *req = NULL; 229 230 list_for_each_entry(timeout, &ctx->timeout_list, list) { 231 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 232 233 if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) && 234 cd->data != tmp->cqe.user_data) 235 continue; 236 if (cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY)) { 237 if (cd->seq == tmp->work.cancel_seq) 238 continue; 239 tmp->work.cancel_seq = cd->seq; 240 } 241 req = tmp; 242 break; 243 } 244 if (!req) 245 return ERR_PTR(-ENOENT); 246 247 io = req->async_data; 248 if (hrtimer_try_to_cancel(&io->timer) == -1) 249 return ERR_PTR(-EALREADY); 250 timeout = io_kiocb_to_cmd(req, struct io_timeout); 251 list_del_init(&timeout->list); 252 return req; 253 } 254 255 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) 256 __must_hold(&ctx->completion_lock) 257 { 258 struct io_kiocb *req; 259 260 spin_lock_irq(&ctx->timeout_lock); 261 req = io_timeout_extract(ctx, cd); 262 spin_unlock_irq(&ctx->timeout_lock); 263 264 if (IS_ERR(req)) 265 return PTR_ERR(req); 266 io_req_task_queue_fail(req, -ECANCELED); 267 return 0; 268 } 269 270 static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) 271 { 272 unsigned issue_flags = *locked ? 0 : IO_URING_F_UNLOCKED; 273 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 274 struct io_kiocb *prev = timeout->prev; 275 int ret = -ENOENT; 276 277 if (prev) { 278 if (!(req->task->flags & PF_EXITING)) { 279 struct io_cancel_data cd = { 280 .ctx = req->ctx, 281 .data = prev->cqe.user_data, 282 }; 283 284 ret = io_try_cancel(req->task->io_uring, &cd, issue_flags); 285 } 286 io_req_set_res(req, ret ?: -ETIME, 0); 287 io_req_task_complete(req, locked); 288 io_put_req(prev); 289 } else { 290 io_req_set_res(req, -ETIME, 0); 291 io_req_task_complete(req, locked); 292 } 293 } 294 295 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) 296 { 297 struct io_timeout_data *data = container_of(timer, 298 struct io_timeout_data, timer); 299 struct io_kiocb *prev, *req = data->req; 300 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 301 struct io_ring_ctx *ctx = req->ctx; 302 unsigned long flags; 303 304 spin_lock_irqsave(&ctx->timeout_lock, flags); 305 prev = timeout->head; 306 timeout->head = NULL; 307 308 /* 309 * We don't expect the list to be empty, that will only happen if we 310 * race with the completion of the linked work. 311 */ 312 if (prev) { 313 io_remove_next_linked(prev); 314 if (!req_ref_inc_not_zero(prev)) 315 prev = NULL; 316 } 317 list_del(&timeout->list); 318 timeout->prev = prev; 319 spin_unlock_irqrestore(&ctx->timeout_lock, flags); 320 321 req->io_task_work.func = io_req_task_link_timeout; 322 io_req_task_work_add(req); 323 return HRTIMER_NORESTART; 324 } 325 326 static clockid_t io_timeout_get_clock(struct io_timeout_data *data) 327 { 328 switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) { 329 case IORING_TIMEOUT_BOOTTIME: 330 return CLOCK_BOOTTIME; 331 case IORING_TIMEOUT_REALTIME: 332 return CLOCK_REALTIME; 333 default: 334 /* can't happen, vetted at prep time */ 335 WARN_ON_ONCE(1); 336 fallthrough; 337 case 0: 338 return CLOCK_MONOTONIC; 339 } 340 } 341 342 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 343 struct timespec64 *ts, enum hrtimer_mode mode) 344 __must_hold(&ctx->timeout_lock) 345 { 346 struct io_timeout_data *io; 347 struct io_timeout *timeout; 348 struct io_kiocb *req = NULL; 349 350 list_for_each_entry(timeout, &ctx->ltimeout_list, list) { 351 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 352 353 if (user_data == tmp->cqe.user_data) { 354 req = tmp; 355 break; 356 } 357 } 358 if (!req) 359 return -ENOENT; 360 361 io = req->async_data; 362 if (hrtimer_try_to_cancel(&io->timer) == -1) 363 return -EALREADY; 364 hrtimer_init(&io->timer, io_timeout_get_clock(io), mode); 365 io->timer.function = io_link_timeout_fn; 366 hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode); 367 return 0; 368 } 369 370 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 371 struct timespec64 *ts, enum hrtimer_mode mode) 372 __must_hold(&ctx->timeout_lock) 373 { 374 struct io_cancel_data cd = { .data = user_data, }; 375 struct io_kiocb *req = io_timeout_extract(ctx, &cd); 376 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 377 struct io_timeout_data *data; 378 379 if (IS_ERR(req)) 380 return PTR_ERR(req); 381 382 timeout->off = 0; /* noseq */ 383 data = req->async_data; 384 list_add_tail(&timeout->list, &ctx->timeout_list); 385 hrtimer_init(&data->timer, io_timeout_get_clock(data), mode); 386 data->timer.function = io_timeout_fn; 387 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode); 388 return 0; 389 } 390 391 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 392 { 393 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); 394 395 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) 396 return -EINVAL; 397 if (sqe->buf_index || sqe->len || sqe->splice_fd_in) 398 return -EINVAL; 399 400 tr->ltimeout = false; 401 tr->addr = READ_ONCE(sqe->addr); 402 tr->flags = READ_ONCE(sqe->timeout_flags); 403 if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) { 404 if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 405 return -EINVAL; 406 if (tr->flags & IORING_LINK_TIMEOUT_UPDATE) 407 tr->ltimeout = true; 408 if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS)) 409 return -EINVAL; 410 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2))) 411 return -EFAULT; 412 if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0) 413 return -EINVAL; 414 } else if (tr->flags) { 415 /* timeout removal doesn't support flags */ 416 return -EINVAL; 417 } 418 419 return 0; 420 } 421 422 static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) 423 { 424 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS 425 : HRTIMER_MODE_REL; 426 } 427 428 /* 429 * Remove or update an existing timeout command 430 */ 431 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) 432 { 433 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); 434 struct io_ring_ctx *ctx = req->ctx; 435 int ret; 436 437 if (!(tr->flags & IORING_TIMEOUT_UPDATE)) { 438 struct io_cancel_data cd = { .data = tr->addr, }; 439 440 spin_lock(&ctx->completion_lock); 441 ret = io_timeout_cancel(ctx, &cd); 442 spin_unlock(&ctx->completion_lock); 443 } else { 444 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags); 445 446 spin_lock_irq(&ctx->timeout_lock); 447 if (tr->ltimeout) 448 ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode); 449 else 450 ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode); 451 spin_unlock_irq(&ctx->timeout_lock); 452 } 453 454 if (ret < 0) 455 req_set_fail(req); 456 io_req_set_res(req, ret, 0); 457 return IOU_OK; 458 } 459 460 static int __io_timeout_prep(struct io_kiocb *req, 461 const struct io_uring_sqe *sqe, 462 bool is_timeout_link) 463 { 464 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 465 struct io_timeout_data *data; 466 unsigned flags; 467 u32 off = READ_ONCE(sqe->off); 468 469 if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in) 470 return -EINVAL; 471 if (off && is_timeout_link) 472 return -EINVAL; 473 flags = READ_ONCE(sqe->timeout_flags); 474 if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK | 475 IORING_TIMEOUT_ETIME_SUCCESS)) 476 return -EINVAL; 477 /* more than one clock specified is invalid, obviously */ 478 if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 479 return -EINVAL; 480 481 INIT_LIST_HEAD(&timeout->list); 482 timeout->off = off; 483 if (unlikely(off && !req->ctx->off_timeout_used)) 484 req->ctx->off_timeout_used = true; 485 486 if (WARN_ON_ONCE(req_has_async_data(req))) 487 return -EFAULT; 488 if (io_alloc_async_data(req)) 489 return -ENOMEM; 490 491 data = req->async_data; 492 data->req = req; 493 data->flags = flags; 494 495 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) 496 return -EFAULT; 497 498 if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0) 499 return -EINVAL; 500 501 INIT_LIST_HEAD(&timeout->list); 502 data->mode = io_translate_timeout_mode(flags); 503 hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode); 504 505 if (is_timeout_link) { 506 struct io_submit_link *link = &req->ctx->submit_state.link; 507 508 if (!link->head) 509 return -EINVAL; 510 if (link->last->opcode == IORING_OP_LINK_TIMEOUT) 511 return -EINVAL; 512 timeout->head = link->last; 513 link->last->flags |= REQ_F_ARM_LTIMEOUT; 514 } 515 return 0; 516 } 517 518 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 519 { 520 return __io_timeout_prep(req, sqe, false); 521 } 522 523 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 524 { 525 return __io_timeout_prep(req, sqe, true); 526 } 527 528 int io_timeout(struct io_kiocb *req, unsigned int issue_flags) 529 { 530 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 531 struct io_ring_ctx *ctx = req->ctx; 532 struct io_timeout_data *data = req->async_data; 533 struct list_head *entry; 534 u32 tail, off = timeout->off; 535 536 spin_lock_irq(&ctx->timeout_lock); 537 538 /* 539 * sqe->off holds how many events that need to occur for this 540 * timeout event to be satisfied. If it isn't set, then this is 541 * a pure timeout request, sequence isn't used. 542 */ 543 if (io_is_timeout_noseq(req)) { 544 entry = ctx->timeout_list.prev; 545 goto add; 546 } 547 548 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); 549 timeout->target_seq = tail + off; 550 551 /* Update the last seq here in case io_flush_timeouts() hasn't. 552 * This is safe because ->completion_lock is held, and submissions 553 * and completions are never mixed in the same ->completion_lock section. 554 */ 555 ctx->cq_last_tm_flush = tail; 556 557 /* 558 * Insertion sort, ensuring the first entry in the list is always 559 * the one we need first. 560 */ 561 list_for_each_prev(entry, &ctx->timeout_list) { 562 struct io_timeout *nextt = list_entry(entry, struct io_timeout, list); 563 struct io_kiocb *nxt = cmd_to_io_kiocb(nextt); 564 565 if (io_is_timeout_noseq(nxt)) 566 continue; 567 /* nxt.seq is behind @tail, otherwise would've been completed */ 568 if (off >= nextt->target_seq - tail) 569 break; 570 } 571 add: 572 list_add(&timeout->list, entry); 573 data->timer.function = io_timeout_fn; 574 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); 575 spin_unlock_irq(&ctx->timeout_lock); 576 return IOU_ISSUE_SKIP_COMPLETE; 577 } 578 579 void io_queue_linked_timeout(struct io_kiocb *req) 580 { 581 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 582 struct io_ring_ctx *ctx = req->ctx; 583 584 spin_lock_irq(&ctx->timeout_lock); 585 /* 586 * If the back reference is NULL, then our linked request finished 587 * before we got a chance to setup the timer 588 */ 589 if (timeout->head) { 590 struct io_timeout_data *data = req->async_data; 591 592 data->timer.function = io_link_timeout_fn; 593 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), 594 data->mode); 595 list_add_tail(&timeout->list, &ctx->ltimeout_list); 596 } 597 spin_unlock_irq(&ctx->timeout_lock); 598 /* drop submission reference */ 599 io_put_req(req); 600 } 601 602 static bool io_match_task(struct io_kiocb *head, struct task_struct *task, 603 bool cancel_all) 604 __must_hold(&req->ctx->timeout_lock) 605 { 606 struct io_kiocb *req; 607 608 if (task && head->task != task) 609 return false; 610 if (cancel_all) 611 return true; 612 613 io_for_each_link(req, head) { 614 if (req->flags & REQ_F_INFLIGHT) 615 return true; 616 } 617 return false; 618 } 619 620 /* Returns true if we found and killed one or more timeouts */ 621 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, 622 bool cancel_all) 623 { 624 struct io_timeout *timeout, *tmp; 625 int canceled = 0; 626 627 io_cq_lock(ctx); 628 spin_lock_irq(&ctx->timeout_lock); 629 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 630 struct io_kiocb *req = cmd_to_io_kiocb(timeout); 631 632 if (io_match_task(req, tsk, cancel_all) && 633 io_kill_timeout(req, -ECANCELED)) 634 canceled++; 635 } 636 spin_unlock_irq(&ctx->timeout_lock); 637 io_cq_unlock_post(ctx); 638 return canceled != 0; 639 } 640