235 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
236 static void pipe_destroy_write_buffer(struct pipe *wpipe);
237 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
238 static void pipe_clone_write_buffer(struct pipe *wpipe);
270 	struct pipe *rpipe, *wpipe;  in pipe_zone_ctor()  local
286 	wpipe = &pp->pp_wpipe;  in pipe_zone_ctor()
287 	bzero(wpipe, sizeof(*wpipe));  in pipe_zone_ctor()
288 	wpipe->pipe_ctime = rpipe->pipe_ctime;  in pipe_zone_ctor()
289 	wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;  in pipe_zone_ctor()
291 	rpipe->pipe_peer = wpipe;  in pipe_zone_ctor()
293 	wpipe->pipe_peer = rpipe;  in pipe_zone_ctor()
294 	wpipe->pipe_pair = pp;  in pipe_zone_ctor()
302 	wpipe->pipe_present = PIPE_ACTIVE;  in pipe_zone_ctor()
343 	struct pipe *rpipe, *wpipe;  in pipe_paircreate()  local
357 	wpipe = &pp->pp_wpipe;  in pipe_paircreate()
360 	knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));  in pipe_paircreate()
369 	error = pipe_create(wpipe, false);  in pipe_paircreate()
385 	wpipe->pipe_state |= PIPE_DIRECTOK;  in pipe_paircreate()
390 	knlist_destroy(&wpipe->pipe_sel.si_note);  in pipe_paircreate()
449 	struct pipe *rpipe, *wpipe;  in kern_pipe()  local
457 	wpipe = &pp->pp_wpipe;  in kern_pipe()
461 		pipeclose(wpipe);  in kern_pipe()
483 		pipeclose(wpipe);  in kern_pipe()
487 	finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);  in kern_pipe()
872 pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio)  in pipe_build_write_buffer()  argument
877 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_build_write_buffer()
878 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_build_write_buffer()
879 	    ("%s: PIPE_DIRECTW set on %p", __func__, wpipe));  in pipe_build_write_buffer()
880 	KASSERT(wpipe->pipe_pages.cnt == 0,  in pipe_build_write_buffer()
881 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_build_write_buffer()
883 	if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)  in pipe_build_write_buffer()
884                 size = wpipe->pipe_buffer.size;  in pipe_build_write_buffer()
888 	wpipe->pipe_state |= PIPE_DIRECTW;  in pipe_build_write_buffer()
889 	PIPE_UNLOCK(wpipe);  in pipe_build_write_buffer()
892 	    wpipe->pipe_pages.ms, PIPENPAGES);  in pipe_build_write_buffer()
893 	PIPE_LOCK(wpipe);  in pipe_build_write_buffer()
895 		wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_build_write_buffer()
899 	wpipe->pipe_pages.npages = i;  in pipe_build_write_buffer()
900 	wpipe->pipe_pages.pos =  in pipe_build_write_buffer()
902 	wpipe->pipe_pages.cnt = size;  in pipe_build_write_buffer()
917 pipe_destroy_write_buffer(struct pipe *wpipe)  in pipe_destroy_write_buffer()  argument
920 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_destroy_write_buffer()
921 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_destroy_write_buffer()
922 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_destroy_write_buffer()
923 	KASSERT(wpipe->pipe_pages.cnt == 0,  in pipe_destroy_write_buffer()
924 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_destroy_write_buffer()
926 	wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_destroy_write_buffer()
927 	vm_page_unhold_pages(wpipe->pipe_pages.ms, wpipe->pipe_pages.npages);  in pipe_destroy_write_buffer()
928 	wpipe->pipe_pages.npages = 0;  in pipe_destroy_write_buffer()
937 pipe_clone_write_buffer(struct pipe *wpipe)  in pipe_clone_write_buffer()  argument
944 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_clone_write_buffer()
945 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_clone_write_buffer()
946 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_clone_write_buffer()
948 	size = wpipe->pipe_pages.cnt;  in pipe_clone_write_buffer()
949 	pos = wpipe->pipe_pages.pos;  in pipe_clone_write_buffer()
950 	wpipe->pipe_pages.cnt = 0;  in pipe_clone_write_buffer()
952 	wpipe->pipe_buffer.in = size;  in pipe_clone_write_buffer()
953 	wpipe->pipe_buffer.out = 0;  in pipe_clone_write_buffer()
954 	wpipe->pipe_buffer.cnt = size;  in pipe_clone_write_buffer()
956 	PIPE_UNLOCK(wpipe);  in pipe_clone_write_buffer()
957 	iov.iov_base = wpipe->pipe_buffer.buffer;  in pipe_clone_write_buffer()
966 	uiomove_fromphys(wpipe->pipe_pages.ms, pos, size, &uio);  in pipe_clone_write_buffer()
967 	PIPE_LOCK(wpipe);  in pipe_clone_write_buffer()
968 	pipe_destroy_write_buffer(wpipe);  in pipe_clone_write_buffer()
979 pipe_direct_write(struct pipe *wpipe, struct uio *uio)  in pipe_direct_write()  argument
984 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_direct_write()
985 	if ((wpipe->pipe_state & PIPE_EOF) != 0) {  in pipe_direct_write()
989 	if (wpipe->pipe_state & PIPE_DIRECTW) {  in pipe_direct_write()
990 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
991 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
992 			wakeup(wpipe);  in pipe_direct_write()
994 		pipeselwakeup(wpipe);  in pipe_direct_write()
995 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
996 		pipeunlock(wpipe);  in pipe_direct_write()
997 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
999 		pipelock(wpipe, 0);  in pipe_direct_write()
1004 	if (wpipe->pipe_buffer.cnt > 0) {  in pipe_direct_write()
1005 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1006 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1007 			wakeup(wpipe);  in pipe_direct_write()
1009 		pipeselwakeup(wpipe);  in pipe_direct_write()
1010 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1011 		pipeunlock(wpipe);  in pipe_direct_write()
1012 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
1014 		pipelock(wpipe, 0);  in pipe_direct_write()
1020 	error = pipe_build_write_buffer(wpipe, uio);  in pipe_direct_write()
1025 	while (wpipe->pipe_pages.cnt != 0 &&  in pipe_direct_write()
1026 	    (wpipe->pipe_state & PIPE_EOF) == 0) {  in pipe_direct_write()
1027 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1028 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1029 			wakeup(wpipe);  in pipe_direct_write()
1031 		pipeselwakeup(wpipe);  in pipe_direct_write()
1032 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1033 		pipeunlock(wpipe);  in pipe_direct_write()
1034 		error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,  in pipe_direct_write()
1036 		pipelock(wpipe, 0);  in pipe_direct_write()
1041 	if ((wpipe->pipe_state & PIPE_EOF) != 0) {  in pipe_direct_write()
1042 		wpipe->pipe_pages.cnt = 0;  in pipe_direct_write()
1043 		pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
1044 		pipeselwakeup(wpipe);  in pipe_direct_write()
1047 		pipe_clone_write_buffer(wpipe);  in pipe_direct_write()
1049 		pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
1051 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_direct_write()
1052 	    ("pipe %p leaked PIPE_DIRECTW", wpipe));  in pipe_direct_write()
1056 	wakeup(wpipe);  in pipe_direct_write()
1065 	struct pipe *wpipe, *rpipe;  in pipe_write()  local
1070 	wpipe = PIPE_PEER(rpipe);  in pipe_write()
1072 	error = pipelock(wpipe, 1);  in pipe_write()
1080 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_write()
1081 	    (wpipe->pipe_state & PIPE_EOF)) {  in pipe_write()
1082 		pipeunlock(wpipe);  in pipe_write()
1087 	error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);  in pipe_write()
1089 		pipeunlock(wpipe);  in pipe_write()
1094 	++wpipe->pipe_busy;  in pipe_write()
1097 	desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);  in pipe_write()
1098 	while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {  in pipe_write()
1110 	    wpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&  in pipe_write()
1111 	    wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&  in pipe_write()
1116 	if (desiredsize != wpipe->pipe_buffer.size &&  in pipe_write()
1117 	    (wpipe->pipe_state & PIPE_DIRECTW) == 0) {  in pipe_write()
1118 		PIPE_UNLOCK(wpipe);  in pipe_write()
1119 		pipespace(wpipe, desiredsize);  in pipe_write()
1120 		PIPE_LOCK(wpipe);  in pipe_write()
1122 	MPASS(wpipe->pipe_buffer.size != 0);  in pipe_write()
1129 		if (wpipe->pipe_state & PIPE_EOF) {  in pipe_write()
1145 		    wpipe->pipe_buffer.size >= PIPE_MINDIRECT &&  in pipe_write()
1147 			error = pipe_direct_write(wpipe, uio);  in pipe_write()
1161 		if (wpipe->pipe_pages.cnt != 0) {  in pipe_write()
1162 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1163 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1164 				wakeup(wpipe);  in pipe_write()
1166 			pipeselwakeup(wpipe);  in pipe_write()
1167 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1168 			pipeunlock(wpipe);  in pipe_write()
1169 			error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,  in pipe_write()
1171 			pipelock(wpipe, 0);  in pipe_write()
1177 		space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;  in pipe_write()
1202 			segsize = wpipe->pipe_buffer.size -  in pipe_write()
1203 				wpipe->pipe_buffer.in;  in pipe_write()
1210 			error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],  in pipe_write()
1215 				KASSERT(wpipe->pipe_buffer.in + segsize ==  in pipe_write()
1216 					wpipe->pipe_buffer.size,  in pipe_write()
1226 				    &wpipe->pipe_buffer.buffer[0],  in pipe_write()
1231 				wpipe->pipe_buffer.in += size;  in pipe_write()
1232 				if (wpipe->pipe_buffer.in >=  in pipe_write()
1233 				    wpipe->pipe_buffer.size) {  in pipe_write()
1234 					KASSERT(wpipe->pipe_buffer.in ==  in pipe_write()
1236 						wpipe->pipe_buffer.size,  in pipe_write()
1238 					wpipe->pipe_buffer.in = size - segsize;  in pipe_write()
1241 				wpipe->pipe_buffer.cnt += size;  in pipe_write()
1242 				KASSERT(wpipe->pipe_buffer.cnt <=  in pipe_write()
1243 					wpipe->pipe_buffer.size,  in pipe_write()
1253 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1254 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1255 				wakeup(wpipe);  in pipe_write()
1270 			pipeselwakeup(wpipe);  in pipe_write()
1272 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1273 			pipeunlock(wpipe);  in pipe_write()
1274 			error = msleep(wpipe, PIPE_MTX(rpipe),  in pipe_write()
1276 			pipelock(wpipe, 0);  in pipe_write()
1283 	--wpipe->pipe_busy;  in pipe_write()
1285 	if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {  in pipe_write()
1286 		wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);  in pipe_write()
1287 		wakeup(wpipe);  in pipe_write()
1288 	} else if (wpipe->pipe_buffer.cnt > 0) {  in pipe_write()
1293 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1294 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1295 			wakeup(wpipe);  in pipe_write()
1309 		pipe_timestamp(&wpipe->pipe_mtime);  in pipe_write()
1315 	if (wpipe->pipe_buffer.cnt)  in pipe_write()
1316 		pipeselwakeup(wpipe);  in pipe_write()
1318 	pipeunlock(wpipe);  in pipe_write()
1420 	struct pipe *wpipe;  in pipe_poll()  local
1428 	wpipe = PIPE_PEER(rpipe);  in pipe_poll()
1440 		if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1441 		    (wpipe->pipe_state & PIPE_EOF) ||  in pipe_poll()
1442 		    ((wpipe->pipe_state & PIPE_DIRECTW) == 0 &&  in pipe_poll()
1443 		     ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||  in pipe_poll()
1444 			 wpipe->pipe_buffer.size == 0)))  in pipe_poll()
1457 			if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1458 			    (wpipe->pipe_state & PIPE_EOF))  in pipe_poll()
1475 		    wpipe->pipe_present == PIPE_ACTIVE) {  in pipe_poll()
1476 			selrecord(td, &wpipe->pipe_sel);  in pipe_poll()
1477 			if (SEL_WAITING(&wpipe->pipe_sel))  in pipe_poll()
1478 				wpipe->pipe_state |= PIPE_SEL;  in pipe_poll()
1777 	struct pipe *wpipe = kn->kn_hook;  in filt_pipewrite()  local
1783 	if (wpipe->pipe_present == PIPE_ACTIVE ||  in filt_pipewrite()
1784 	    (wpipe->pipe_type & PIPE_TYPE_NAMED) != 0) {  in filt_pipewrite()
1785 		PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in filt_pipewrite()
1787 		if (wpipe->pipe_state & PIPE_DIRECTW) {  in filt_pipewrite()
1789 		} else if (wpipe->pipe_buffer.size > 0) {  in filt_pipewrite()
1790 			kn->kn_data = wpipe->pipe_buffer.size -  in filt_pipewrite()
1791 			    wpipe->pipe_buffer.cnt;  in filt_pipewrite()
1797 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in filt_pipewrite()
1798 	    (wpipe->pipe_state & PIPE_EOF)) {  in filt_pipewrite()