24 	struct netfs_io_request *rreq;  in netfs_alloc_request()  local
30 		rreq = mempool_alloc(mempool, GFP_KERNEL);  in netfs_alloc_request()
31 		if (rreq)  in netfs_alloc_request()
36 	memset(rreq, 0, kmem_cache_size(cache));  in netfs_alloc_request()
37 	rreq->start	= start;  in netfs_alloc_request()
38 	rreq->len	= len;  in netfs_alloc_request()
39 	rreq->origin	= origin;  in netfs_alloc_request()
40 	rreq->netfs_ops	= ctx->ops;  in netfs_alloc_request()
41 	rreq->mapping	= mapping;  in netfs_alloc_request()
42 	rreq->inode	= inode;  in netfs_alloc_request()
43 	rreq->i_size	= i_size_read(inode);  in netfs_alloc_request()
44 	rreq->debug_id	= atomic_inc_return(&debug_ids);  in netfs_alloc_request()
45 	rreq->wsize	= INT_MAX;  in netfs_alloc_request()
46 	rreq->io_streams[0].sreq_max_len = ULONG_MAX;  in netfs_alloc_request()
47 	rreq->io_streams[0].sreq_max_segs = 0;  in netfs_alloc_request()
48 	spin_lock_init(&rreq->lock);  in netfs_alloc_request()
49 	INIT_LIST_HEAD(&rreq->io_streams[0].subrequests);  in netfs_alloc_request()
50 	INIT_LIST_HEAD(&rreq->io_streams[1].subrequests);  in netfs_alloc_request()
51 	init_waitqueue_head(&rreq->waitq);  in netfs_alloc_request()
52 	refcount_set(&rreq->ref, 1);  in netfs_alloc_request()
60 		INIT_WORK(&rreq->work, netfs_read_collection_worker);  in netfs_alloc_request()
61 		rreq->io_streams[0].avail = true;  in netfs_alloc_request()
63 		INIT_WORK(&rreq->work, netfs_write_collection_worker);  in netfs_alloc_request()
66 	__set_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags);  in netfs_alloc_request()
68 		__set_bit(NETFS_RREQ_NONBLOCK, &rreq->flags);  in netfs_alloc_request()
69 	if (rreq->netfs_ops->init_request) {  in netfs_alloc_request()
70 		ret = rreq->netfs_ops->init_request(rreq, file);  in netfs_alloc_request()
72 			mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool);  in netfs_alloc_request()
78 	trace_netfs_rreq_ref(rreq->debug_id, 1, netfs_rreq_trace_new);  in netfs_alloc_request()
79 	netfs_proc_add_rreq(rreq);  in netfs_alloc_request()
81 	return rreq;  in netfs_alloc_request()
84 void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what)  in netfs_get_request()  argument
88 	__refcount_inc(&rreq->ref, &r);  in netfs_get_request()
89 	trace_netfs_rreq_ref(rreq->debug_id, r + 1, what);  in netfs_get_request()
92 void netfs_clear_subrequests(struct netfs_io_request *rreq, bool was_async)  in netfs_clear_subrequests()  argument
98 	for (s = 0; s < ARRAY_SIZE(rreq->io_streams); s++) {  in netfs_clear_subrequests()
99 		stream = &rreq->io_streams[s];  in netfs_clear_subrequests()
112 	struct netfs_io_request *rreq = container_of(rcu, struct netfs_io_request, rcu);  in netfs_free_request_rcu()  local
114 	mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool);  in netfs_free_request_rcu()
120 	struct netfs_io_request *rreq =  in netfs_free_request()  local
122 	struct netfs_inode *ictx = netfs_inode(rreq->inode);  in netfs_free_request()
125 	trace_netfs_rreq(rreq, netfs_rreq_trace_free);  in netfs_free_request()
126 	netfs_proc_del_rreq(rreq);  in netfs_free_request()
127 	netfs_clear_subrequests(rreq, false);  in netfs_free_request()
128 	if (rreq->netfs_ops->free_request)  in netfs_free_request()
129 		rreq->netfs_ops->free_request(rreq);  in netfs_free_request()
130 	if (rreq->cache_resources.ops)  in netfs_free_request()
131 		rreq->cache_resources.ops->end_operation(&rreq->cache_resources);  in netfs_free_request()
132 	if (rreq->direct_bv) {  in netfs_free_request()
133 		for (i = 0; i < rreq->direct_bv_count; i++) {  in netfs_free_request()
134 			if (rreq->direct_bv[i].bv_page) {  in netfs_free_request()
135 				if (rreq->direct_bv_unpin)  in netfs_free_request()
136 					unpin_user_page(rreq->direct_bv[i].bv_page);  in netfs_free_request()
139 		kvfree(rreq->direct_bv);  in netfs_free_request()
141 	rolling_buffer_clear(&rreq->buffer);  in netfs_free_request()
145 	call_rcu(&rreq->rcu, netfs_free_request_rcu);  in netfs_free_request()
148 void netfs_put_request(struct netfs_io_request *rreq, bool was_async,  in netfs_put_request()  argument
155 	if (rreq) {  in netfs_put_request()
156 		debug_id = rreq->debug_id;  in netfs_put_request()
157 		dead = __refcount_dec_and_test(&rreq->ref, &r);  in netfs_put_request()
161 				rreq->work.func = netfs_free_request;  in netfs_put_request()
162 				if (!queue_work(system_unbound_wq, &rreq->work))  in netfs_put_request()
165 				netfs_free_request(&rreq->work);  in netfs_put_request()
174 struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq)  in netfs_alloc_subrequest()  argument
177 	mempool_t *mempool = rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool;  in netfs_alloc_subrequest()
181 		subreq = mempool_alloc(rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool,  in netfs_alloc_subrequest()
192 	subreq->rreq = rreq;  in netfs_alloc_subrequest()
193 	subreq->debug_index = atomic_inc_return(&rreq->subreq_counter);  in netfs_alloc_subrequest()
194 	netfs_get_request(rreq, netfs_rreq_trace_get_subreq);  in netfs_alloc_subrequest()
205 	trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index, r + 1,  in netfs_get_subrequest()
212 	struct netfs_io_request *rreq = subreq->rreq;  in netfs_free_subrequest()  local
215 	if (rreq->netfs_ops->free_subrequest)  in netfs_free_subrequest()
216 		rreq->netfs_ops->free_subrequest(subreq);  in netfs_free_subrequest()
217 	mempool_free(subreq, rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool);  in netfs_free_subrequest()
219 	netfs_put_request(rreq, was_async, netfs_rreq_trace_put_subreq);  in netfs_free_subrequest()
226 	unsigned int debug_id = subreq->rreq->debug_id;  in netfs_put_subrequest()