234 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
235 static void pipe_destroy_write_buffer(struct pipe *wpipe);
236 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
237 static void pipe_clone_write_buffer(struct pipe *wpipe);
269 	struct pipe *rpipe, *wpipe;  in pipe_zone_ctor()  local
285 	wpipe = &pp->pp_wpipe;  in pipe_zone_ctor()
286 	bzero(wpipe, sizeof(*wpipe));  in pipe_zone_ctor()
287 	wpipe->pipe_ctime = rpipe->pipe_ctime;  in pipe_zone_ctor()
288 	wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;  in pipe_zone_ctor()
290 	rpipe->pipe_peer = wpipe;  in pipe_zone_ctor()
292 	wpipe->pipe_peer = rpipe;  in pipe_zone_ctor()
293 	wpipe->pipe_pair = pp;  in pipe_zone_ctor()
301 	wpipe->pipe_present = PIPE_ACTIVE;  in pipe_zone_ctor()
342 	struct pipe *rpipe, *wpipe;  in pipe_paircreate()  local
355 	wpipe = &pp->pp_wpipe;  in pipe_paircreate()
358 	knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));  in pipe_paircreate()
362 	pipe_create(wpipe, 0);  in pipe_paircreate()
365 	wpipe->pipe_state |= PIPE_DIRECTOK;  in pipe_paircreate()
403 	struct pipe *rpipe, *wpipe;  in kern_pipe()  local
409 	wpipe = &pp->pp_wpipe;  in kern_pipe()
413 		pipeclose(wpipe);  in kern_pipe()
435 		pipeclose(wpipe);  in kern_pipe()
439 	finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);  in kern_pipe()
817 pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio)  in pipe_build_write_buffer()  argument
822 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_build_write_buffer()
823 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_build_write_buffer()
824 	    ("%s: PIPE_DIRECTW set on %p", __func__, wpipe));  in pipe_build_write_buffer()
825 	KASSERT(wpipe->pipe_map.cnt == 0,  in pipe_build_write_buffer()
826 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_build_write_buffer()
828 	if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)  in pipe_build_write_buffer()
829                 size = wpipe->pipe_buffer.size;  in pipe_build_write_buffer()
833 	wpipe->pipe_state |= PIPE_DIRECTW;  in pipe_build_write_buffer()
834 	PIPE_UNLOCK(wpipe);  in pipe_build_write_buffer()
837 	    wpipe->pipe_map.ms, PIPENPAGES);  in pipe_build_write_buffer()
838 	PIPE_LOCK(wpipe);  in pipe_build_write_buffer()
840 		wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_build_write_buffer()
844 	wpipe->pipe_map.npages = i;  in pipe_build_write_buffer()
845 	wpipe->pipe_map.pos =  in pipe_build_write_buffer()
847 	wpipe->pipe_map.cnt = size;  in pipe_build_write_buffer()
862 pipe_destroy_write_buffer(struct pipe *wpipe)  in pipe_destroy_write_buffer()  argument
865 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_destroy_write_buffer()
866 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_destroy_write_buffer()
867 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_destroy_write_buffer()
868 	KASSERT(wpipe->pipe_map.cnt == 0,  in pipe_destroy_write_buffer()
869 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_destroy_write_buffer()
871 	wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_destroy_write_buffer()
872 	vm_page_unhold_pages(wpipe->pipe_map.ms, wpipe->pipe_map.npages);  in pipe_destroy_write_buffer()
873 	wpipe->pipe_map.npages = 0;  in pipe_destroy_write_buffer()
882 pipe_clone_write_buffer(struct pipe *wpipe)  in pipe_clone_write_buffer()  argument
889 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_clone_write_buffer()
890 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_clone_write_buffer()
891 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_clone_write_buffer()
893 	size = wpipe->pipe_map.cnt;  in pipe_clone_write_buffer()
894 	pos = wpipe->pipe_map.pos;  in pipe_clone_write_buffer()
895 	wpipe->pipe_map.cnt = 0;  in pipe_clone_write_buffer()
897 	wpipe->pipe_buffer.in = size;  in pipe_clone_write_buffer()
898 	wpipe->pipe_buffer.out = 0;  in pipe_clone_write_buffer()
899 	wpipe->pipe_buffer.cnt = size;  in pipe_clone_write_buffer()
901 	PIPE_UNLOCK(wpipe);  in pipe_clone_write_buffer()
902 	iov.iov_base = wpipe->pipe_buffer.buffer;  in pipe_clone_write_buffer()
911 	uiomove_fromphys(wpipe->pipe_map.ms, pos, size, &uio);  in pipe_clone_write_buffer()
912 	PIPE_LOCK(wpipe);  in pipe_clone_write_buffer()
913 	pipe_destroy_write_buffer(wpipe);  in pipe_clone_write_buffer()
924 pipe_direct_write(struct pipe *wpipe, struct uio *uio)  in pipe_direct_write()  argument
929 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_direct_write()
930 	error = pipelock(wpipe, 1);  in pipe_direct_write()
933 	if ((wpipe->pipe_state & PIPE_EOF) != 0) {  in pipe_direct_write()
935 		pipeunlock(wpipe);  in pipe_direct_write()
938 	if (wpipe->pipe_state & PIPE_DIRECTW) {  in pipe_direct_write()
939 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
940 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
941 			wakeup(wpipe);  in pipe_direct_write()
943 		pipeselwakeup(wpipe);  in pipe_direct_write()
944 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
945 		pipeunlock(wpipe);  in pipe_direct_write()
946 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
953 	if (wpipe->pipe_buffer.cnt > 0) {  in pipe_direct_write()
954 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
955 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
956 			wakeup(wpipe);  in pipe_direct_write()
958 		pipeselwakeup(wpipe);  in pipe_direct_write()
959 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
960 		pipeunlock(wpipe);  in pipe_direct_write()
961 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
969 	error = pipe_build_write_buffer(wpipe, uio);  in pipe_direct_write()
971 		pipeunlock(wpipe);  in pipe_direct_write()
975 	while (wpipe->pipe_map.cnt != 0) {  in pipe_direct_write()
976 		if (wpipe->pipe_state & PIPE_EOF) {  in pipe_direct_write()
977 			wpipe->pipe_map.cnt = 0;  in pipe_direct_write()
978 			pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
979 			pipeselwakeup(wpipe);  in pipe_direct_write()
980 			pipeunlock(wpipe);  in pipe_direct_write()
984 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
985 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
986 			wakeup(wpipe);  in pipe_direct_write()
988 		pipeselwakeup(wpipe);  in pipe_direct_write()
989 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
990 		pipeunlock(wpipe);  in pipe_direct_write()
991 		error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,  in pipe_direct_write()
993 		pipelock(wpipe, 0);  in pipe_direct_write()
998 	if (wpipe->pipe_state & PIPE_EOF)  in pipe_direct_write()
1001 		pipe_clone_write_buffer(wpipe);  in pipe_direct_write()
1003 		pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
1004 	pipeunlock(wpipe);  in pipe_direct_write()
1005 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_direct_write()
1006 	    ("pipe %p leaked PIPE_DIRECTW", wpipe));  in pipe_direct_write()
1010 	wakeup(wpipe);  in pipe_direct_write()
1022 	struct pipe *wpipe, *rpipe;  in pipe_write()  local
1025 	wpipe = PIPE_PEER(rpipe);  in pipe_write()
1027 	error = pipelock(wpipe, 1);  in pipe_write()
1035 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_write()
1036 	    (wpipe->pipe_state & PIPE_EOF)) {  in pipe_write()
1037 		pipeunlock(wpipe);  in pipe_write()
1042 	error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);  in pipe_write()
1044 		pipeunlock(wpipe);  in pipe_write()
1049 	++wpipe->pipe_busy;  in pipe_write()
1052 	desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);  in pipe_write()
1053 	while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {  in pipe_write()
1065 		(wpipe->pipe_buffer.size > SMALL_PIPE_SIZE) &&  in pipe_write()
1066 		(wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) &&  in pipe_write()
1071 	if ((desiredsize != wpipe->pipe_buffer.size) &&  in pipe_write()
1072 		((wpipe->pipe_state & PIPE_DIRECTW) == 0)) {  in pipe_write()
1073 		PIPE_UNLOCK(wpipe);  in pipe_write()
1074 		pipespace(wpipe, desiredsize);  in pipe_write()
1075 		PIPE_LOCK(wpipe);  in pipe_write()
1077 	if (wpipe->pipe_buffer.size == 0) {  in pipe_write()
1083 		--wpipe->pipe_busy;  in pipe_write()
1084 		pipeunlock(wpipe);  in pipe_write()
1085 		PIPE_UNLOCK(wpipe);  in pipe_write()
1089 	pipeunlock(wpipe);  in pipe_write()
1096 		pipelock(wpipe, 0);  in pipe_write()
1097 		if (wpipe->pipe_state & PIPE_EOF) {  in pipe_write()
1098 			pipeunlock(wpipe);  in pipe_write()
1114 		    wpipe->pipe_buffer.size >= PIPE_MINDIRECT &&  in pipe_write()
1116 			pipeunlock(wpipe);  in pipe_write()
1117 			error = pipe_direct_write(wpipe, uio);  in pipe_write()
1131 		if (wpipe->pipe_map.cnt != 0) {  in pipe_write()
1132 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1133 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1134 				wakeup(wpipe);  in pipe_write()
1136 			pipeselwakeup(wpipe);  in pipe_write()
1137 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1138 			pipeunlock(wpipe);  in pipe_write()
1139 			error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,  in pipe_write()
1147 		space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;  in pipe_write()
1172 			segsize = wpipe->pipe_buffer.size -  in pipe_write()
1173 				wpipe->pipe_buffer.in;  in pipe_write()
1180 			error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],  in pipe_write()
1185 				KASSERT(wpipe->pipe_buffer.in + segsize ==  in pipe_write()
1186 					wpipe->pipe_buffer.size,  in pipe_write()
1196 				    &wpipe->pipe_buffer.buffer[0],  in pipe_write()
1201 				wpipe->pipe_buffer.in += size;  in pipe_write()
1202 				if (wpipe->pipe_buffer.in >=  in pipe_write()
1203 				    wpipe->pipe_buffer.size) {  in pipe_write()
1204 					KASSERT(wpipe->pipe_buffer.in ==  in pipe_write()
1206 						wpipe->pipe_buffer.size,  in pipe_write()
1208 					wpipe->pipe_buffer.in = size - segsize;  in pipe_write()
1211 				wpipe->pipe_buffer.cnt += size;  in pipe_write()
1212 				KASSERT(wpipe->pipe_buffer.cnt <=  in pipe_write()
1213 					wpipe->pipe_buffer.size,  in pipe_write()
1216 			pipeunlock(wpipe);  in pipe_write()
1223 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1224 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1225 				wakeup(wpipe);  in pipe_write()
1233 				pipeunlock(wpipe);  in pipe_write()
1241 			pipeselwakeup(wpipe);  in pipe_write()
1243 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1244 			pipeunlock(wpipe);  in pipe_write()
1245 			error = msleep(wpipe, PIPE_MTX(rpipe),  in pipe_write()
1252 	pipelock(wpipe, 0);  in pipe_write()
1253 	--wpipe->pipe_busy;  in pipe_write()
1255 	if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {  in pipe_write()
1256 		wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);  in pipe_write()
1257 		wakeup(wpipe);  in pipe_write()
1258 	} else if (wpipe->pipe_buffer.cnt > 0) {  in pipe_write()
1263 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1264 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1265 			wakeup(wpipe);  in pipe_write()
1279 		vfs_timestamp(&wpipe->pipe_mtime);  in pipe_write()
1285 	if (wpipe->pipe_buffer.cnt)  in pipe_write()
1286 		pipeselwakeup(wpipe);  in pipe_write()
1288 	pipeunlock(wpipe);  in pipe_write()
1389 	struct pipe *wpipe;  in pipe_poll()  local
1397 	wpipe = PIPE_PEER(rpipe);  in pipe_poll()
1409 		if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1410 		    (wpipe->pipe_state & PIPE_EOF) ||  in pipe_poll()
1411 		    ((wpipe->pipe_state & PIPE_DIRECTW) == 0 &&  in pipe_poll()
1412 		     ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||  in pipe_poll()
1413 			 wpipe->pipe_buffer.size == 0)))  in pipe_poll()
1425 			if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1426 			    (wpipe->pipe_state & PIPE_EOF))  in pipe_poll()
1439 			selrecord(td, &wpipe->pipe_sel);  in pipe_poll()
1440 			if (SEL_WAITING(&wpipe->pipe_sel))  in pipe_poll()
1441 				wpipe->pipe_state |= PIPE_SEL;  in pipe_poll()
1723 	struct pipe *wpipe = rpipe->pipe_peer;  in filt_piperead()  local
1732 	    wpipe->pipe_present != PIPE_ACTIVE ||  in filt_piperead()
1733 	    (wpipe->pipe_state & PIPE_EOF)) {  in filt_piperead()
1745 	struct pipe *wpipe;  in filt_pipewrite()  local
1751 	wpipe = kn->kn_hook;  in filt_pipewrite()
1752 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in filt_pipewrite()
1753 	    (wpipe->pipe_state & PIPE_EOF)) {  in filt_pipewrite()
1758 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in filt_pipewrite()
1759 	kn->kn_data = (wpipe->pipe_buffer.size > 0) ?  in filt_pipewrite()
1760 	    (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) : PIPE_BUF;  in filt_pipewrite()
1761 	if (wpipe->pipe_state & PIPE_DIRECTW)  in filt_pipewrite()