Lines Matching refs:rw
91 static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw) in tg_bps_limit() argument
98 return tg->bps[rw]; in tg_bps_limit()
101 static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw) in tg_iops_limit() argument
108 return tg->iops[rw]; in tg_iops_limit()
241 int rw; in throtl_pd_alloc() local
255 for (rw = READ; rw <= WRITE; rw++) { in throtl_pd_alloc()
256 throtl_qnode_init(&tg->qnode_on_self[rw], tg); in throtl_pd_alloc()
257 throtl_qnode_init(&tg->qnode_on_parent[rw], tg); in throtl_pd_alloc()
310 int rw; in tg_update_has_rules() local
312 for (rw = READ; rw <= WRITE; rw++) { in tg_update_has_rules()
313 tg->has_rules_iops[rw] = in tg_update_has_rules()
314 (parent_tg && parent_tg->has_rules_iops[rw]) || in tg_update_has_rules()
315 tg_iops_limit(tg, rw) != UINT_MAX; in tg_update_has_rules()
316 tg->has_rules_bps[rw] = in tg_update_has_rules()
317 (parent_tg && parent_tg->has_rules_bps[rw]) || in tg_update_has_rules()
318 tg_bps_limit(tg, rw) != U64_MAX; in tg_update_has_rules()
477 bool rw, unsigned long start) in throtl_start_new_slice_with_credit() argument
479 tg->bytes_disp[rw] = 0; in throtl_start_new_slice_with_credit()
480 tg->io_disp[rw] = 0; in throtl_start_new_slice_with_credit()
488 if (time_after(start, tg->slice_start[rw])) in throtl_start_new_slice_with_credit()
489 tg->slice_start[rw] = start; in throtl_start_new_slice_with_credit()
491 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice_with_credit()
494 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice_with_credit()
495 tg->slice_end[rw], jiffies); in throtl_start_new_slice_with_credit()
498 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw, in throtl_start_new_slice() argument
502 tg->bytes_disp[rw] = 0; in throtl_start_new_slice()
503 tg->io_disp[rw] = 0; in throtl_start_new_slice()
505 tg->slice_start[rw] = jiffies; in throtl_start_new_slice()
506 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice()
510 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice()
511 tg->slice_end[rw], jiffies); in throtl_start_new_slice()
514 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw, in throtl_set_slice_end() argument
517 tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice); in throtl_set_slice_end()
520 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw, in throtl_extend_slice() argument
523 throtl_set_slice_end(tg, rw, jiffy_end); in throtl_extend_slice()
526 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_extend_slice()
527 tg->slice_end[rw], jiffies); in throtl_extend_slice()
531 static bool throtl_slice_used(struct throtl_grp *tg, bool rw) in throtl_slice_used() argument
533 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw])) in throtl_slice_used()
575 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) in throtl_trim_slice() argument
581 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw])); in throtl_trim_slice()
588 if (throtl_slice_used(tg, rw)) in throtl_trim_slice()
598 throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice); in throtl_trim_slice()
600 time_elapsed = rounddown(jiffies - tg->slice_start[rw], in throtl_trim_slice()
615 bytes_trim = calculate_bytes_allowed(tg_bps_limit(tg, rw), in throtl_trim_slice()
617 io_trim = calculate_io_allowed(tg_iops_limit(tg, rw), time_elapsed); in throtl_trim_slice()
621 if ((long long)tg->bytes_disp[rw] >= bytes_trim) in throtl_trim_slice()
622 tg->bytes_disp[rw] -= bytes_trim; in throtl_trim_slice()
624 tg->bytes_disp[rw] = 0; in throtl_trim_slice()
626 if ((int)tg->io_disp[rw] >= io_trim) in throtl_trim_slice()
627 tg->io_disp[rw] -= io_trim; in throtl_trim_slice()
629 tg->io_disp[rw] = 0; in throtl_trim_slice()
631 tg->slice_start[rw] += time_elapsed; in throtl_trim_slice()
635 rw == READ ? 'R' : 'W', time_elapsed / tg->td->throtl_slice, in throtl_trim_slice()
636 bytes_trim, io_trim, tg->slice_start[rw], tg->slice_end[rw], in throtl_trim_slice()
640 static void __tg_update_carryover(struct throtl_grp *tg, bool rw, in __tg_update_carryover() argument
643 unsigned long jiffy_elapsed = jiffies - tg->slice_start[rw]; in __tg_update_carryover()
644 u64 bps_limit = tg_bps_limit(tg, rw); in __tg_update_carryover()
645 u32 iops_limit = tg_iops_limit(tg, rw); in __tg_update_carryover()
655 tg->bytes_disp[rw]; in __tg_update_carryover()
658 tg->io_disp[rw]; in __tg_update_carryover()
659 tg->bytes_disp[rw] -= *bytes; in __tg_update_carryover()
660 tg->io_disp[rw] -= *ios; in __tg_update_carryover()
681 bool rw = bio_data_dir(bio); in tg_within_iops_limit() local
689 jiffy_elapsed = jiffies - tg->slice_start[rw]; in tg_within_iops_limit()
694 if (io_allowed > 0 && tg->io_disp[rw] + 1 <= io_allowed) in tg_within_iops_limit()
708 bool rw = bio_data_dir(bio); in tg_within_bps_limit() local
719 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; in tg_within_bps_limit()
727 if (bytes_allowed > 0 && tg->bytes_disp[rw] + bio_size <= bytes_allowed) in tg_within_bps_limit()
731 extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed; in tg_within_bps_limit()
752 bool rw = bio_data_dir(bio); in tg_may_dispatch() local
754 u64 bps_limit = tg_bps_limit(tg, rw); in tg_may_dispatch()
755 u32 iops_limit = tg_iops_limit(tg, rw); in tg_may_dispatch()
763 BUG_ON(tg->service_queue.nr_queued[rw] && in tg_may_dispatch()
764 bio != throtl_peek_queued(&tg->service_queue.queued[rw])); in tg_may_dispatch()
781 if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw])) in tg_may_dispatch()
782 throtl_start_new_slice(tg, rw, true); in tg_may_dispatch()
784 if (time_before(tg->slice_end[rw], in tg_may_dispatch()
786 throtl_extend_slice(tg, rw, in tg_may_dispatch()
803 if (time_before(tg->slice_end[rw], jiffies + max_wait)) in tg_may_dispatch()
804 throtl_extend_slice(tg, rw, jiffies + max_wait); in tg_may_dispatch()
811 bool rw = bio_data_dir(bio); in throtl_charge_bio() local
816 tg->bytes_disp[rw] += bio_size; in throtl_charge_bio()
818 tg->io_disp[rw]++; in throtl_charge_bio()
834 bool rw = bio_data_dir(bio); in throtl_add_bio_tg() local
837 qn = &tg->qnode_on_self[rw]; in throtl_add_bio_tg()
845 if (!sq->nr_queued[rw]) in throtl_add_bio_tg()
848 throtl_qnode_add_bio(bio, qn, &sq->queued[rw]); in throtl_add_bio_tg()
850 sq->nr_queued[rw]++; in throtl_add_bio_tg()
881 struct throtl_grp *parent_tg, bool rw) in start_parent_slice_with_credit() argument
883 if (throtl_slice_used(parent_tg, rw)) { in start_parent_slice_with_credit()
884 throtl_start_new_slice_with_credit(parent_tg, rw, in start_parent_slice_with_credit()
885 child_tg->slice_start[rw]); in start_parent_slice_with_credit()
890 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw) in tg_dispatch_one_bio() argument
904 bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put); in tg_dispatch_one_bio()
905 sq->nr_queued[rw]--; in tg_dispatch_one_bio()
917 throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg); in tg_dispatch_one_bio()
918 start_parent_slice_with_credit(tg, parent_tg, rw); in tg_dispatch_one_bio()
921 throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw], in tg_dispatch_one_bio()
922 &parent_sq->queued[rw]); in tg_dispatch_one_bio()
923 BUG_ON(tg->td->nr_queued[rw] <= 0); in tg_dispatch_one_bio()
924 tg->td->nr_queued[rw]--; in tg_dispatch_one_bio()
927 throtl_trim_slice(tg, rw); in tg_dispatch_one_bio()
1098 int rw; in blk_throtl_dispatch_work_fn() local
1103 for (rw = READ; rw <= WRITE; rw++) in blk_throtl_dispatch_work_fn()
1104 while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL))) in blk_throtl_dispatch_work_fn()
1607 static bool tg_within_limit(struct throtl_grp *tg, struct bio *bio, bool rw) in tg_within_limit() argument
1610 if (tg->service_queue.nr_queued[rw]) in tg_within_limit()
1623 bool rw = bio_data_dir(bio); in __blk_throtl_bio() local
1632 if (tg_within_limit(tg, bio, rw)) { in __blk_throtl_bio()
1647 throtl_trim_slice(tg, rw); in __blk_throtl_bio()
1668 qn = &tg->qnode_on_parent[rw]; in __blk_throtl_bio()
1679 rw == READ ? 'R' : 'W', in __blk_throtl_bio()
1680 tg->bytes_disp[rw], bio->bi_iter.bi_size, in __blk_throtl_bio()
1681 tg_bps_limit(tg, rw), in __blk_throtl_bio()
1682 tg->io_disp[rw], tg_iops_limit(tg, rw), in __blk_throtl_bio()
1685 td->nr_queued[rw]++; in __blk_throtl_bio()