235 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
236 static void pipe_destroy_write_buffer(struct pipe *wpipe);
237 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
238 static void pipe_clone_write_buffer(struct pipe *wpipe);
270 	struct pipe *rpipe, *wpipe;  in pipe_zone_ctor()  local
286 	wpipe = &pp->pp_wpipe;  in pipe_zone_ctor()
287 	bzero(wpipe, sizeof(*wpipe));  in pipe_zone_ctor()
288 	wpipe->pipe_ctime = rpipe->pipe_ctime;  in pipe_zone_ctor()
289 	wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;  in pipe_zone_ctor()
291 	rpipe->pipe_peer = wpipe;  in pipe_zone_ctor()
293 	wpipe->pipe_peer = rpipe;  in pipe_zone_ctor()
294 	wpipe->pipe_pair = pp;  in pipe_zone_ctor()
302 	wpipe->pipe_present = PIPE_ACTIVE;  in pipe_zone_ctor()
343 	struct pipe *rpipe, *wpipe;  in pipe_paircreate()  local
357 	wpipe = &pp->pp_wpipe;  in pipe_paircreate()
360 	knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));  in pipe_paircreate()
369 	error = pipe_create(wpipe, false);  in pipe_paircreate()
385 	wpipe->pipe_state |= PIPE_DIRECTOK;  in pipe_paircreate()
390 	knlist_destroy(&wpipe->pipe_sel.si_note);  in pipe_paircreate()
448 	struct pipe *rpipe, *wpipe;  in kern_pipe()  local
456 	wpipe = &pp->pp_wpipe;  in kern_pipe()
460 		pipeclose(wpipe);  in kern_pipe()
482 		pipeclose(wpipe);  in kern_pipe()
486 	finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);  in kern_pipe()
871 pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio)  in pipe_build_write_buffer()  argument
876 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_build_write_buffer()
877 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_build_write_buffer()
878 	    ("%s: PIPE_DIRECTW set on %p", __func__, wpipe));  in pipe_build_write_buffer()
879 	KASSERT(wpipe->pipe_pages.cnt == 0,  in pipe_build_write_buffer()
880 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_build_write_buffer()
882 	if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)  in pipe_build_write_buffer()
883                 size = wpipe->pipe_buffer.size;  in pipe_build_write_buffer()
887 	wpipe->pipe_state |= PIPE_DIRECTW;  in pipe_build_write_buffer()
888 	PIPE_UNLOCK(wpipe);  in pipe_build_write_buffer()
891 	    wpipe->pipe_pages.ms, PIPENPAGES);  in pipe_build_write_buffer()
892 	PIPE_LOCK(wpipe);  in pipe_build_write_buffer()
894 		wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_build_write_buffer()
898 	wpipe->pipe_pages.npages = i;  in pipe_build_write_buffer()
899 	wpipe->pipe_pages.pos =  in pipe_build_write_buffer()
901 	wpipe->pipe_pages.cnt = size;  in pipe_build_write_buffer()
916 pipe_destroy_write_buffer(struct pipe *wpipe)  in pipe_destroy_write_buffer()  argument
919 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_destroy_write_buffer()
920 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_destroy_write_buffer()
921 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_destroy_write_buffer()
922 	KASSERT(wpipe->pipe_pages.cnt == 0,  in pipe_destroy_write_buffer()
923 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_destroy_write_buffer()
925 	wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_destroy_write_buffer()
926 	vm_page_unhold_pages(wpipe->pipe_pages.ms, wpipe->pipe_pages.npages);  in pipe_destroy_write_buffer()
927 	wpipe->pipe_pages.npages = 0;  in pipe_destroy_write_buffer()
936 pipe_clone_write_buffer(struct pipe *wpipe)  in pipe_clone_write_buffer()  argument
943 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_clone_write_buffer()
944 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_clone_write_buffer()
945 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_clone_write_buffer()
947 	size = wpipe->pipe_pages.cnt;  in pipe_clone_write_buffer()
948 	pos = wpipe->pipe_pages.pos;  in pipe_clone_write_buffer()
949 	wpipe->pipe_pages.cnt = 0;  in pipe_clone_write_buffer()
951 	wpipe->pipe_buffer.in = size;  in pipe_clone_write_buffer()
952 	wpipe->pipe_buffer.out = 0;  in pipe_clone_write_buffer()
953 	wpipe->pipe_buffer.cnt = size;  in pipe_clone_write_buffer()
955 	PIPE_UNLOCK(wpipe);  in pipe_clone_write_buffer()
956 	iov.iov_base = wpipe->pipe_buffer.buffer;  in pipe_clone_write_buffer()
965 	uiomove_fromphys(wpipe->pipe_pages.ms, pos, size, &uio);  in pipe_clone_write_buffer()
966 	PIPE_LOCK(wpipe);  in pipe_clone_write_buffer()
967 	pipe_destroy_write_buffer(wpipe);  in pipe_clone_write_buffer()
978 pipe_direct_write(struct pipe *wpipe, struct uio *uio)  in pipe_direct_write()  argument
983 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_direct_write()
984 	if ((wpipe->pipe_state & PIPE_EOF) != 0) {  in pipe_direct_write()
988 	if (wpipe->pipe_state & PIPE_DIRECTW) {  in pipe_direct_write()
989 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
990 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
991 			wakeup(wpipe);  in pipe_direct_write()
993 		pipeselwakeup(wpipe);  in pipe_direct_write()
994 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
995 		pipeunlock(wpipe);  in pipe_direct_write()
996 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
998 		pipelock(wpipe, 0);  in pipe_direct_write()
1003 	if (wpipe->pipe_buffer.cnt > 0) {  in pipe_direct_write()
1004 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1005 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1006 			wakeup(wpipe);  in pipe_direct_write()
1008 		pipeselwakeup(wpipe);  in pipe_direct_write()
1009 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1010 		pipeunlock(wpipe);  in pipe_direct_write()
1011 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
1013 		pipelock(wpipe, 0);  in pipe_direct_write()
1019 	error = pipe_build_write_buffer(wpipe, uio);  in pipe_direct_write()
1024 	while (wpipe->pipe_pages.cnt != 0 &&  in pipe_direct_write()
1025 	    (wpipe->pipe_state & PIPE_EOF) == 0) {  in pipe_direct_write()
1026 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1027 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1028 			wakeup(wpipe);  in pipe_direct_write()
1030 		pipeselwakeup(wpipe);  in pipe_direct_write()
1031 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1032 		pipeunlock(wpipe);  in pipe_direct_write()
1033 		error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,  in pipe_direct_write()
1035 		pipelock(wpipe, 0);  in pipe_direct_write()
1040 	if ((wpipe->pipe_state & PIPE_EOF) != 0) {  in pipe_direct_write()
1041 		wpipe->pipe_pages.cnt = 0;  in pipe_direct_write()
1042 		pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
1043 		pipeselwakeup(wpipe);  in pipe_direct_write()
1046 		pipe_clone_write_buffer(wpipe);  in pipe_direct_write()
1048 		pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
1050 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_direct_write()
1051 	    ("pipe %p leaked PIPE_DIRECTW", wpipe));  in pipe_direct_write()
1055 	wakeup(wpipe);  in pipe_direct_write()
1064 	struct pipe *wpipe, *rpipe;  in pipe_write()  local
1069 	wpipe = PIPE_PEER(rpipe);  in pipe_write()
1071 	error = pipelock(wpipe, 1);  in pipe_write()
1079 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_write()
1080 	    (wpipe->pipe_state & PIPE_EOF)) {  in pipe_write()
1081 		pipeunlock(wpipe);  in pipe_write()
1086 	error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);  in pipe_write()
1088 		pipeunlock(wpipe);  in pipe_write()
1093 	++wpipe->pipe_busy;  in pipe_write()
1096 	desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);  in pipe_write()
1097 	while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {  in pipe_write()
1109 	    wpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&  in pipe_write()
1110 	    wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&  in pipe_write()
1115 	if (desiredsize != wpipe->pipe_buffer.size &&  in pipe_write()
1116 	    (wpipe->pipe_state & PIPE_DIRECTW) == 0) {  in pipe_write()
1117 		PIPE_UNLOCK(wpipe);  in pipe_write()
1118 		pipespace(wpipe, desiredsize);  in pipe_write()
1119 		PIPE_LOCK(wpipe);  in pipe_write()
1121 	MPASS(wpipe->pipe_buffer.size != 0);  in pipe_write()
1128 		if (wpipe->pipe_state & PIPE_EOF) {  in pipe_write()
1144 		    wpipe->pipe_buffer.size >= PIPE_MINDIRECT &&  in pipe_write()
1146 			error = pipe_direct_write(wpipe, uio);  in pipe_write()
1160 		if (wpipe->pipe_pages.cnt != 0) {  in pipe_write()
1161 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1162 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1163 				wakeup(wpipe);  in pipe_write()
1165 			pipeselwakeup(wpipe);  in pipe_write()
1166 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1167 			pipeunlock(wpipe);  in pipe_write()
1168 			error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,  in pipe_write()
1170 			pipelock(wpipe, 0);  in pipe_write()
1176 		space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;  in pipe_write()
1201 			segsize = wpipe->pipe_buffer.size -  in pipe_write()
1202 				wpipe->pipe_buffer.in;  in pipe_write()
1209 			error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],  in pipe_write()
1214 				KASSERT(wpipe->pipe_buffer.in + segsize ==  in pipe_write()
1215 					wpipe->pipe_buffer.size,  in pipe_write()
1225 				    &wpipe->pipe_buffer.buffer[0],  in pipe_write()
1230 				wpipe->pipe_buffer.in += size;  in pipe_write()
1231 				if (wpipe->pipe_buffer.in >=  in pipe_write()
1232 				    wpipe->pipe_buffer.size) {  in pipe_write()
1233 					KASSERT(wpipe->pipe_buffer.in ==  in pipe_write()
1235 						wpipe->pipe_buffer.size,  in pipe_write()
1237 					wpipe->pipe_buffer.in = size - segsize;  in pipe_write()
1240 				wpipe->pipe_buffer.cnt += size;  in pipe_write()
1241 				KASSERT(wpipe->pipe_buffer.cnt <=  in pipe_write()
1242 					wpipe->pipe_buffer.size,  in pipe_write()
1252 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1253 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1254 				wakeup(wpipe);  in pipe_write()
1269 			pipeselwakeup(wpipe);  in pipe_write()
1271 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1272 			pipeunlock(wpipe);  in pipe_write()
1273 			error = msleep(wpipe, PIPE_MTX(rpipe),  in pipe_write()
1275 			pipelock(wpipe, 0);  in pipe_write()
1282 	--wpipe->pipe_busy;  in pipe_write()
1284 	if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {  in pipe_write()
1285 		wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);  in pipe_write()
1286 		wakeup(wpipe);  in pipe_write()
1287 	} else if (wpipe->pipe_buffer.cnt > 0) {  in pipe_write()
1292 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1293 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1294 			wakeup(wpipe);  in pipe_write()
1308 		pipe_timestamp(&wpipe->pipe_mtime);  in pipe_write()
1314 	if (wpipe->pipe_buffer.cnt)  in pipe_write()
1315 		pipeselwakeup(wpipe);  in pipe_write()
1317 	pipeunlock(wpipe);  in pipe_write()
1419 	struct pipe *wpipe;  in pipe_poll()  local
1427 	wpipe = PIPE_PEER(rpipe);  in pipe_poll()
1439 		if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1440 		    (wpipe->pipe_state & PIPE_EOF) ||  in pipe_poll()
1441 		    ((wpipe->pipe_state & PIPE_DIRECTW) == 0 &&  in pipe_poll()
1442 		     ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||  in pipe_poll()
1443 			 wpipe->pipe_buffer.size == 0)))  in pipe_poll()
1456 			if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1457 			    (wpipe->pipe_state & PIPE_EOF))  in pipe_poll()
1474 			selrecord(td, &wpipe->pipe_sel);  in pipe_poll()
1475 			if (SEL_WAITING(&wpipe->pipe_sel))  in pipe_poll()
1476 				wpipe->pipe_state |= PIPE_SEL;  in pipe_poll()
1775 	struct pipe *wpipe = kn->kn_hook;  in filt_pipewrite()  local
1781 	if (wpipe->pipe_present == PIPE_ACTIVE ||  in filt_pipewrite()
1782 	    (wpipe->pipe_type & PIPE_TYPE_NAMED) != 0) {  in filt_pipewrite()
1783 		PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in filt_pipewrite()
1785 		if (wpipe->pipe_state & PIPE_DIRECTW) {  in filt_pipewrite()
1787 		} else if (wpipe->pipe_buffer.size > 0) {  in filt_pipewrite()
1788 			kn->kn_data = wpipe->pipe_buffer.size -  in filt_pipewrite()
1789 			    wpipe->pipe_buffer.cnt;  in filt_pipewrite()
1795 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in filt_pipewrite()
1796 	    (wpipe->pipe_state & PIPE_EOF)) {  in filt_pipewrite()