78 static void bch2_open_bucket_hash_add(struct bch_fs *c, struct open_bucket *ob)  in bch2_open_bucket_hash_add()  argument
80 	open_bucket_idx_t idx = ob - c->open_buckets;  in bch2_open_bucket_hash_add()
81 	open_bucket_idx_t *slot = open_bucket_hashslot(c, ob->dev, ob->bucket);  in bch2_open_bucket_hash_add()
83 	ob->hash = *slot;  in bch2_open_bucket_hash_add()
87 static void bch2_open_bucket_hash_remove(struct bch_fs *c, struct open_bucket *ob)  in bch2_open_bucket_hash_remove()  argument
89 	open_bucket_idx_t idx = ob - c->open_buckets;  in bch2_open_bucket_hash_remove()
90 	open_bucket_idx_t *slot = open_bucket_hashslot(c, ob->dev, ob->bucket);  in bch2_open_bucket_hash_remove()
97 	*slot = ob->hash;  in bch2_open_bucket_hash_remove()
98 	ob->hash = 0;  in bch2_open_bucket_hash_remove()
101 void __bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob)  in __bch2_open_bucket_put()  argument
103 	struct bch_dev *ca = ob_dev(c, ob);  in __bch2_open_bucket_put()
105 	if (ob->ec) {  in __bch2_open_bucket_put()
106 		ec_stripe_new_put(c, ob->ec, STRIPE_REF_io);  in __bch2_open_bucket_put()
110 	spin_lock(&ob->lock);  in __bch2_open_bucket_put()
111 	ob->valid = false;  in __bch2_open_bucket_put()
112 	ob->data_type = 0;  in __bch2_open_bucket_put()
113 	spin_unlock(&ob->lock);  in __bch2_open_bucket_put()
116 	bch2_open_bucket_hash_remove(c, ob);  in __bch2_open_bucket_put()
118 	ob->freelist = c->open_buckets_freelist;  in __bch2_open_bucket_put()
119 	c->open_buckets_freelist = ob - c->open_buckets;  in __bch2_open_bucket_put()
132 	struct open_bucket *ob;  in bch2_open_bucket_write_error()  local
135 	open_bucket_for_each(c, obs, ob, i)  in bch2_open_bucket_write_error()
136 		if (ob->dev == dev && ob->ec)  in bch2_open_bucket_write_error()
137 			bch2_ec_bucket_cancel(c, ob, err);  in bch2_open_bucket_write_error()
142 	struct open_bucket *ob;  in bch2_open_bucket_alloc()  local
146 	ob = c->open_buckets + c->open_buckets_freelist;  in bch2_open_bucket_alloc()
147 	c->open_buckets_freelist = ob->freelist;  in bch2_open_bucket_alloc()
148 	atomic_set(&ob->pin, 1);  in bch2_open_bucket_alloc()
149 	ob->data_type = 0;  in bch2_open_bucket_alloc()
152 	return ob;  in bch2_open_bucket_alloc()
163 static void open_bucket_free_unused(struct bch_fs *c, struct open_bucket *ob)  in open_bucket_free_unused()  argument
170 	bch2_dev_rcu(c, ob->dev)->nr_partial_buckets++;  in open_bucket_free_unused()
173 	ob->on_partial_list = true;  in open_bucket_free_unused()
175 		ob - c->open_buckets;  in open_bucket_free_unused()
241 	struct open_bucket *ob = bch2_open_bucket_alloc(c);  in __try_alloc_bucket()  local
243 	spin_lock(&ob->lock);  in __try_alloc_bucket()
244 	ob->valid	= true;  in __try_alloc_bucket()
245 	ob->sectors_free = ca->mi.bucket_size;  in __try_alloc_bucket()
246 	ob->dev		= ca->dev_idx;  in __try_alloc_bucket()
247 	ob->gen		= gen;  in __try_alloc_bucket()
248 	ob->bucket	= bucket;  in __try_alloc_bucket()
249 	spin_unlock(&ob->lock);  in __try_alloc_bucket()
252 	bch2_open_bucket_hash_add(c, ob);  in __try_alloc_bucket()
258 	return ob;  in __try_alloc_bucket()
296 	struct open_bucket *ob = NULL;  in bch2_bucket_alloc_early()  local
353 		ob = may_alloc_bucket(c, k.k->p, s)  in bch2_bucket_alloc_early()
360 		if (ob)  in bch2_bucket_alloc_early()
367 	if (!ob && ret)  in bch2_bucket_alloc_early()
368 		ob = ERR_PTR(ret);  in bch2_bucket_alloc_early()
370 	if (!ob && alloc_start > first_bucket) {  in bch2_bucket_alloc_early()
377 	return ob;  in bch2_bucket_alloc_early()
388 	struct open_bucket *ob = NULL;  in bch2_bucket_alloc_freelist()  local
425 			ob = try_alloc_bucket(trans, ca, watermark, s, &iter, cl);  in bch2_bucket_alloc_freelist()
426 			if (ob) {  in bch2_bucket_alloc_freelist()
427 				if (!IS_ERR(ob))  in bch2_bucket_alloc_freelist()
437 		if (ob || ret)  in bch2_bucket_alloc_freelist()
443 	BUG_ON(ob && ret);  in bch2_bucket_alloc_freelist()
446 		ob = ERR_PTR(ret);  in bch2_bucket_alloc_freelist()
448 	if (!ob && alloc_start > ca->mi.first_bucket) {  in bch2_bucket_alloc_freelist()
453 	return ob;  in bch2_bucket_alloc_freelist()
462 					 struct open_bucket *ob)  in trace_bucket_alloc2()  argument
484 	if (!IS_ERR(ob)) {  in trace_bucket_alloc2()
485 		prt_printf(&buf, "allocated\t%llu\n", ob->bucket);  in trace_bucket_alloc2()
488 		prt_printf(&buf, "err\t%s\n", bch2_err_str(PTR_ERR(ob)));  in trace_bucket_alloc2()
516 	struct open_bucket *ob = NULL;  in bch2_bucket_alloc_trans()  local
549 		ob = ERR_PTR(-BCH_ERR_freelist_empty);  in bch2_bucket_alloc_trans()
556 	ob = likely(freespace)  in bch2_bucket_alloc_trans()
563 	if (!ob && s.btree_bitmap != BTREE_BITMAP_ANY) {  in bch2_bucket_alloc_trans()
568 	if (!ob && freespace && c->curr_recovery_pass <= BCH_RECOVERY_PASS_check_alloc_info) {  in bch2_bucket_alloc_trans()
573 	if (!ob)  in bch2_bucket_alloc_trans()
574 		ob = ERR_PTR(-BCH_ERR_no_buckets_found);  in bch2_bucket_alloc_trans()
576 	if (!IS_ERR(ob))  in bch2_bucket_alloc_trans()
577 		ob->data_type = data_type;  in bch2_bucket_alloc_trans()
579 	if (!IS_ERR(ob))  in bch2_bucket_alloc_trans()
581 	else if (!bch2_err_matches(PTR_ERR(ob), BCH_ERR_transaction_restart))  in bch2_bucket_alloc_trans()
584 	if (!IS_ERR(ob)  in bch2_bucket_alloc_trans()
587 		trace_bucket_alloc2(c, ca, watermark, data_type, cl, usage, &s, ob);  in bch2_bucket_alloc_trans()
589 	return ob;  in bch2_bucket_alloc_trans()
598 	struct open_bucket *ob;  in bch2_bucket_alloc()  local
601 		      PTR_ERR_OR_ZERO(ob = bch2_bucket_alloc_trans(trans, ca, watermark,  in bch2_bucket_alloc()
603 	return ob;  in bch2_bucket_alloc()
701 			   struct open_bucket *ob)  in add_new_bucket()  argument
703 	unsigned durability = ob_dev(c, ob)->mi.durability;  in add_new_bucket()
707 	__clear_bit(ob->dev, devs_may_alloc->d);  in add_new_bucket()
711 	ob_push(c, ptrs, ob);  in add_new_bucket()
715 	if (ob->ec)  in add_new_bucket()
749 		struct open_bucket *ob = bch2_bucket_alloc_trans(trans, ca, watermark, data_type,  in bch2_bucket_alloc_set_trans()  local
751 		if (!IS_ERR(ob))  in bch2_bucket_alloc_set_trans()
755 		if (IS_ERR(ob)) {  in bch2_bucket_alloc_set_trans()
756 			ret = PTR_ERR(ob);  in bch2_bucket_alloc_set_trans()
764 				   have_cache, ob)) {  in bch2_bucket_alloc_set_trans()
815 			struct open_bucket *ob = c->open_buckets + h->s->blocks[ec_idx];  in bucket_alloc_from_stripe()  local
816 			if (ob->dev == *i && !test_and_set_bit(ec_idx, h->s->blocks_allocated)) {  in bucket_alloc_from_stripe()
817 				ob->ec_idx	= ec_idx;  in bucket_alloc_from_stripe()
818 				ob->ec		= h->s;  in bucket_alloc_from_stripe()
823 						     have_cache, ob);  in bucket_alloc_from_stripe()
838 			struct open_bucket *ob)  in want_bucket()  argument
840 	struct bch_dev *ca = ob_dev(c, ob);  in want_bucket()
842 	if (!test_bit(ob->dev, devs_may_alloc->d))  in want_bucket()
845 	if (ob->data_type != wp->data_type)  in want_bucket()
852 	if (ec != (ob->ec != NULL))  in want_bucket()
868 	struct open_bucket *ob;  in bucket_alloc_set_writepoint()  local
872 	open_bucket_for_each(c, &wp->ptrs, ob, i) {  in bucket_alloc_set_writepoint()
874 					have_cache, ec, ob))  in bucket_alloc_set_writepoint()
877 				       have_cache, ob);  in bucket_alloc_set_writepoint()
879 			ob_push(c, &ptrs_skip, ob);  in bucket_alloc_set_writepoint()
906 		struct open_bucket *ob = c->open_buckets + c->open_buckets_partial[i];  in bucket_alloc_set_partial()  local
908 		if (want_bucket(c, wp, devs_may_alloc, have_cache, ec, ob)) {  in bucket_alloc_set_partial()
909 			struct bch_dev *ca = ob_dev(c, ob);  in bucket_alloc_set_partial()
921 			ob->on_partial_list = false;  in bucket_alloc_set_partial()
924 			bch2_dev_rcu(c, ob->dev)->nr_partial_buckets--;  in bucket_alloc_set_partial()
929 					     have_cache, ob);  in bucket_alloc_set_partial()
954 	struct open_bucket *ob;  in __open_bucket_add_buckets()  local
965 	open_bucket_for_each(c, ptrs, ob, i)  in __open_bucket_add_buckets()
966 		__clear_bit(ob->dev, devs.d);  in __open_bucket_add_buckets()
1056 static bool should_drop_bucket(struct open_bucket *ob, struct bch_fs *c,  in should_drop_bucket()  argument
1060 		return ob->ec != NULL;  in should_drop_bucket()
1062 		bool drop = ob->dev == ca->dev_idx;  in should_drop_bucket()
1066 		if (!drop && ob->ec) {  in should_drop_bucket()
1069 			mutex_lock(&ob->ec->lock);  in should_drop_bucket()
1070 			nr_blocks = bkey_i_to_stripe(&ob->ec->new_stripe.key)->v.nr_blocks;  in should_drop_bucket()
1073 				if (!ob->ec->blocks[i])  in should_drop_bucket()
1076 				ob2 = c->open_buckets + ob->ec->blocks[i];  in should_drop_bucket()
1079 			mutex_unlock(&ob->ec->lock);  in should_drop_bucket()
1092 	struct open_bucket *ob;  in bch2_writepoint_stop()  local
1096 	open_bucket_for_each(c, &wp->ptrs, ob, i)  in bch2_writepoint_stop()
1097 		if (should_drop_bucket(ob, c, ca, ec))  in bch2_writepoint_stop()
1098 			bch2_open_bucket_put(c, ob);  in bch2_writepoint_stop()
1100 			ob_push(c, &ptrs, ob);  in bch2_writepoint_stop()
1123 		bch2_open_buckets_put(c, &a->ob);  in bch2_open_buckets_stop()
1130 		struct open_bucket *ob =  in bch2_open_buckets_stop()  local
1133 		if (should_drop_bucket(ob, c, ca, ec)) {  in bch2_open_buckets_stop()
1138 			ob->on_partial_list = false;  in bch2_open_buckets_stop()
1141 			bch2_dev_rcu(c, ob->dev)->nr_partial_buckets--;  in bch2_open_buckets_stop()
1145 			bch2_open_bucket_put(c, ob);  in bch2_open_buckets_stop()
1205 	struct open_bucket *ob;  in try_decrease_writepoints()  local
1226 	open_bucket_for_each(c, &wp->ptrs, ob, i)  in try_decrease_writepoints()
1227 		open_bucket_free_unused(c, ob);  in try_decrease_writepoints()
1297 	struct open_bucket *ob;  in deallocate_extra_replicas()  local
1300 	open_bucket_for_each(c, ptrs, ob, i) {  in deallocate_extra_replicas()
1301 		unsigned d = ob_dev(c, ob)->mi.durability;  in deallocate_extra_replicas()
1305 			ob_push(c, ptrs_no_use, ob);  in deallocate_extra_replicas()
1307 			ob_push(c, &ptrs2, ob);  in deallocate_extra_replicas()
1331 	struct open_bucket *ob;  in bch2_alloc_sectors_start_trans()  local
1418 	open_bucket_for_each(c, &wp->ptrs, ob, i)  in bch2_alloc_sectors_start_trans()
1419 		open_bucket_free_unused(c, ob);  in bch2_alloc_sectors_start_trans()
1425 	open_bucket_for_each(c, &wp->ptrs, ob, i) {  in bch2_alloc_sectors_start_trans()
1431 		struct bch_dev *ca = ob_dev(c, ob);  in bch2_alloc_sectors_start_trans()
1432 		u64 offset = bucket_to_sector(ca, ob->bucket) +  in bch2_alloc_sectors_start_trans()
1434 			ob->sectors_free;  in bch2_alloc_sectors_start_trans()
1437 		ob->sectors_free = max_t(int, 0, ob->sectors_free - align);  in bch2_alloc_sectors_start_trans()
1439 		wp->sectors_free = min(wp->sectors_free, ob->sectors_free);  in bch2_alloc_sectors_start_trans()
1454 	open_bucket_for_each(c, &wp->ptrs, ob, i)  in bch2_alloc_sectors_start_trans()
1456 			ob_push(c, &ptrs, ob);  in bch2_alloc_sectors_start_trans()
1458 			open_bucket_free_unused(c, ob);  in bch2_alloc_sectors_start_trans()
1477 struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *c, struct open_bucket *ob)  in bch2_ob_ptr()  argument
1479 	struct bch_dev *ca = ob_dev(c, ob);  in bch2_ob_ptr()
1483 		.gen	= ob->gen,  in bch2_ob_ptr()
1484 		.dev	= ob->dev,  in bch2_ob_ptr()
1485 		.offset	= bucket_to_sector(ca, ob->bucket) +  in bch2_ob_ptr()
1487 			ob->sectors_free,  in bch2_ob_ptr()
1520 	struct open_bucket *ob;  in bch2_fs_allocator_foreground_init()  local
1529 	for (ob = c->open_buckets + 1;  in bch2_fs_allocator_foreground_init()
1530 	     ob < c->open_buckets + ARRAY_SIZE(c->open_buckets); ob++) {  in bch2_fs_allocator_foreground_init()
1531 		spin_lock_init(&ob->lock);  in bch2_fs_allocator_foreground_init()
1534 		ob->freelist = c->open_buckets_freelist;  in bch2_fs_allocator_foreground_init()
1535 		c->open_buckets_freelist = ob - c->open_buckets;  in bch2_fs_allocator_foreground_init()
1553 void bch2_open_bucket_to_text(struct printbuf *out, struct bch_fs *c, struct open_bucket *ob)  in bch2_open_bucket_to_text()  argument
1555 	struct bch_dev *ca = ob_dev(c, ob);  in bch2_open_bucket_to_text()
1556 	unsigned data_type = ob->data_type;  in bch2_open_bucket_to_text()
1560 		   ob - c->open_buckets,  in bch2_open_bucket_to_text()
1561 		   atomic_read(&ob->pin));  in bch2_open_bucket_to_text()
1564 		   ob->dev, ob->bucket, ob->gen,  in bch2_open_bucket_to_text()
1565 		   ca->mi.bucket_size - ob->sectors_free, ca->mi.bucket_size);  in bch2_open_bucket_to_text()
1566 	if (ob->ec)  in bch2_open_bucket_to_text()
1567 		prt_printf(out, " ec idx %llu", ob->ec->idx);  in bch2_open_bucket_to_text()
1568 	if (ob->on_partial_list)  in bch2_open_bucket_to_text()
1576 	struct open_bucket *ob;  in bch2_open_buckets_to_text()  local
1580 	for (ob = c->open_buckets;  in bch2_open_buckets_to_text()
1581 	     ob < c->open_buckets + ARRAY_SIZE(c->open_buckets);  in bch2_open_buckets_to_text()
1582 	     ob++) {  in bch2_open_buckets_to_text()
1583 		spin_lock(&ob->lock);  in bch2_open_buckets_to_text()
1584 		if (ob->valid && (!ca || ob->dev == ca->dev_idx))  in bch2_open_buckets_to_text()
1585 			bch2_open_bucket_to_text(out, c, ob);  in bch2_open_buckets_to_text()
1586 		spin_unlock(&ob->lock);  in bch2_open_buckets_to_text()
1617 	struct open_bucket *ob;  in bch2_write_point_to_text()  local
1634 	open_bucket_for_each(c, &wp->ptrs, ob, i)  in bch2_write_point_to_text()
1635 		bch2_open_bucket_to_text(out, c, ob);  in bch2_write_point_to_text()