78 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_check_timeout()  local
86 	spin_lock(&fiq->lock);  in fuse_check_timeout()
87 	expired = fuse_request_expired(fc, &fiq->pending);  in fuse_check_timeout()
88 	spin_unlock(&fiq->lock);  in fuse_check_timeout()
302 static u64 fuse_get_unique_locked(struct fuse_iqueue *fiq)  in fuse_get_unique_locked()  argument
304 	fiq->reqctr += FUSE_REQ_ID_STEP;  in fuse_get_unique_locked()
305 	return fiq->reqctr;  in fuse_get_unique_locked()
308 u64 fuse_get_unique(struct fuse_iqueue *fiq)  in fuse_get_unique()  argument
312 	spin_lock(&fiq->lock);  in fuse_get_unique()
313 	ret = fuse_get_unique_locked(fiq);  in fuse_get_unique()
314 	spin_unlock(&fiq->lock);  in fuse_get_unique()
328 static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq)  in fuse_dev_wake_and_unlock()  argument
329 __releases(fiq->lock)  in fuse_dev_wake_and_unlock()
331 	wake_up(&fiq->waitq);  in fuse_dev_wake_and_unlock()
332 	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in fuse_dev_wake_and_unlock()
333 	spin_unlock(&fiq->lock);  in fuse_dev_wake_and_unlock()
336 void fuse_dev_queue_forget(struct fuse_iqueue *fiq,  in fuse_dev_queue_forget()  argument
339 	spin_lock(&fiq->lock);  in fuse_dev_queue_forget()
340 	if (fiq->connected) {  in fuse_dev_queue_forget()
341 		fiq->forget_list_tail->next = forget;  in fuse_dev_queue_forget()
342 		fiq->forget_list_tail = forget;  in fuse_dev_queue_forget()
343 		fuse_dev_wake_and_unlock(fiq);  in fuse_dev_queue_forget()
346 		spin_unlock(&fiq->lock);  in fuse_dev_queue_forget()
350 void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)  in fuse_dev_queue_interrupt()  argument
352 	spin_lock(&fiq->lock);  in fuse_dev_queue_interrupt()
354 		list_add_tail(&req->intr_entry, &fiq->interrupts);  in fuse_dev_queue_interrupt()
362 			spin_unlock(&fiq->lock);  in fuse_dev_queue_interrupt()
364 			fuse_dev_wake_and_unlock(fiq);  in fuse_dev_queue_interrupt()
367 		spin_unlock(&fiq->lock);  in fuse_dev_queue_interrupt()
371 static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req)  in fuse_dev_queue_req()  argument
373 	spin_lock(&fiq->lock);  in fuse_dev_queue_req()
374 	if (fiq->connected) {  in fuse_dev_queue_req()
376 			req->in.h.unique = fuse_get_unique_locked(fiq);  in fuse_dev_queue_req()
377 		list_add_tail(&req->list, &fiq->pending);  in fuse_dev_queue_req()
378 		fuse_dev_wake_and_unlock(fiq);  in fuse_dev_queue_req()
380 		spin_unlock(&fiq->lock);  in fuse_dev_queue_req()
394 static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req)  in fuse_send_one()  argument
400 	fiq->ops->send_req(fiq, req);  in fuse_send_one()
406 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_queue_forget()  local
411 	fiq->ops->send_forget(fiq, forget);  in fuse_queue_forget()
416 	struct fuse_iqueue *fiq = &fc->iq;  in flush_bg_queue()  local
425 		fuse_send_one(fiq, req);  in flush_bg_queue()
441 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_request_end()  local
453 		spin_lock(&fiq->lock);  in fuse_request_end()
455 		spin_unlock(&fiq->lock);  in fuse_request_end()
494 	struct fuse_iqueue *fiq = &req->fm->fc->iq;  in queue_interrupt()  local
500 	fiq->ops->send_interrupt(fiq, req);  in queue_interrupt()
526 	struct fuse_iqueue *fiq = &fc->iq;  in request_wait_answer()  local
555 			removed = fuse_remove_pending_req(req, &fiq->lock);  in request_wait_answer()
569 	struct fuse_iqueue *fiq = &req->fm->fc->iq;  in __fuse_request_send()  local
576 	fuse_send_one(fiq, req);  in __fuse_request_send()
687 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_request_queue_background_uring()  local
689 	req->in.h.unique = fuse_get_unique(fiq);  in fuse_request_queue_background_uring()
766 	struct fuse_iqueue *fiq = &fm->fc->iq;  in fuse_simple_notify_reply()  local
777 	fuse_send_one(fiq, req);  in fuse_simple_notify_reply()
1207 static int forget_pending(struct fuse_iqueue *fiq)  in forget_pending()  argument
1209 	return fiq->forget_list_head.next != NULL;  in forget_pending()
1212 static int request_pending(struct fuse_iqueue *fiq)  in request_pending()  argument
1214 	return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||  in request_pending()
1215 		forget_pending(fiq);  in request_pending()
1226 static int fuse_read_interrupt(struct fuse_iqueue *fiq,  in fuse_read_interrupt()  argument
1229 __releases(fiq->lock)  in fuse_read_interrupt()
1244 	spin_unlock(&fiq->lock);  in fuse_read_interrupt()
1256 static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,  in fuse_dequeue_forget()  argument
1260 	struct fuse_forget_link *head = fiq->forget_list_head.next;  in fuse_dequeue_forget()
1267 	fiq->forget_list_head.next = *newhead;  in fuse_dequeue_forget()
1269 	if (fiq->forget_list_head.next == NULL)  in fuse_dequeue_forget()
1270 		fiq->forget_list_tail = &fiq->forget_list_head;  in fuse_dequeue_forget()
1278 static int fuse_read_single_forget(struct fuse_iqueue *fiq,  in fuse_read_single_forget()  argument
1281 __releases(fiq->lock)  in fuse_read_single_forget()
1284 	struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL);  in fuse_read_single_forget()
1291 		.unique = fuse_get_unique_locked(fiq),  in fuse_read_single_forget()
1295 	spin_unlock(&fiq->lock);  in fuse_read_single_forget()
1311 static int fuse_read_batch_forget(struct fuse_iqueue *fiq,  in fuse_read_batch_forget()  argument
1313 __releases(fiq->lock)  in fuse_read_batch_forget()
1322 		.unique = fuse_get_unique_locked(fiq),  in fuse_read_batch_forget()
1327 		spin_unlock(&fiq->lock);  in fuse_read_batch_forget()
1332 	head = fuse_dequeue_forget(fiq, max_forgets, &count);  in fuse_read_batch_forget()
1333 	spin_unlock(&fiq->lock);  in fuse_read_batch_forget()
1360 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,  in fuse_read_forget()  argument
1363 __releases(fiq->lock)  in fuse_read_forget()
1365 	if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)  in fuse_read_forget()
1366 		return fuse_read_single_forget(fiq, cs, nbytes);  in fuse_read_forget()
1368 		return fuse_read_batch_forget(fiq, cs, nbytes);  in fuse_read_forget()
1385 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_dev_do_read()  local
1412 		spin_lock(&fiq->lock);  in fuse_dev_do_read()
1413 		if (!fiq->connected || request_pending(fiq))  in fuse_dev_do_read()
1415 		spin_unlock(&fiq->lock);  in fuse_dev_do_read()
1419 		err = wait_event_interruptible_exclusive(fiq->waitq,  in fuse_dev_do_read()
1420 				!fiq->connected || request_pending(fiq));  in fuse_dev_do_read()
1425 	if (!fiq->connected) {  in fuse_dev_do_read()
1430 	if (!list_empty(&fiq->interrupts)) {  in fuse_dev_do_read()
1431 		req = list_entry(fiq->interrupts.next, struct fuse_req,  in fuse_dev_do_read()
1433 		return fuse_read_interrupt(fiq, cs, nbytes, req);  in fuse_dev_do_read()
1436 	if (forget_pending(fiq)) {  in fuse_dev_do_read()
1437 		if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)  in fuse_dev_do_read()
1438 			return fuse_read_forget(fc, fiq, cs, nbytes);  in fuse_dev_do_read()
1440 		if (fiq->forget_batch <= -8)  in fuse_dev_do_read()
1441 			fiq->forget_batch = 16;  in fuse_dev_do_read()
1444 	req = list_entry(fiq->pending.next, struct fuse_req, list);  in fuse_dev_do_read()
1447 	spin_unlock(&fiq->lock);  in fuse_dev_do_read()
1514 	spin_unlock(&fiq->lock);  in fuse_dev_do_read()
1978 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_resend()  local
2005 	spin_lock(&fiq->lock);  in fuse_resend()
2006 	if (!fiq->connected) {  in fuse_resend()
2007 		spin_unlock(&fiq->lock);  in fuse_resend()
2014 	list_splice(&to_queue, &fiq->pending);  in fuse_resend()
2015 	fuse_dev_wake_and_unlock(fiq);  in fuse_resend()
2315 	struct fuse_iqueue *fiq;  in fuse_dev_poll()  local
2321 	fiq = &fud->fc->iq;  in fuse_dev_poll()
2322 	poll_wait(file, &fiq->waitq, wait);  in fuse_dev_poll()
2324 	spin_lock(&fiq->lock);  in fuse_dev_poll()
2325 	if (!fiq->connected)  in fuse_dev_poll()
2327 	else if (request_pending(fiq))  in fuse_dev_poll()
2329 	spin_unlock(&fiq->lock);  in fuse_dev_poll()
2382 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_abort_conn()  local
2427 		spin_lock(&fiq->lock);  in fuse_abort_conn()
2428 		fiq->connected = 0;  in fuse_abort_conn()
2429 		list_for_each_entry(req, &fiq->pending, list)  in fuse_abort_conn()
2431 		list_splice_tail_init(&fiq->pending, &to_end);  in fuse_abort_conn()
2432 		while (forget_pending(fiq))  in fuse_abort_conn()
2433 			kfree(fuse_dequeue_forget(fiq, 1, NULL));  in fuse_abort_conn()
2434 		wake_up_all(&fiq->waitq);  in fuse_abort_conn()
2435 		spin_unlock(&fiq->lock);  in fuse_abort_conn()
2436 		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in fuse_abort_conn()