238 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
239 static void pipe_destroy_write_buffer(struct pipe *wpipe);
240 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
241 static void pipe_clone_write_buffer(struct pipe *wpipe);
296 	struct pipe *rpipe, *wpipe;  in pipe_zone_ctor()  local
312 	wpipe = &pp->pp_wpipe;  in pipe_zone_ctor()
313 	bzero(wpipe, sizeof(*wpipe));  in pipe_zone_ctor()
314 	wpipe->pipe_ctime = rpipe->pipe_ctime;  in pipe_zone_ctor()
315 	wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;  in pipe_zone_ctor()
317 	rpipe->pipe_peer = wpipe;  in pipe_zone_ctor()
319 	wpipe->pipe_peer = rpipe;  in pipe_zone_ctor()
320 	wpipe->pipe_pair = pp;  in pipe_zone_ctor()
328 	wpipe->pipe_present = PIPE_ACTIVE;  in pipe_zone_ctor()
369 	struct pipe *rpipe, *wpipe;  in pipe_paircreate()  local
383 	wpipe = &pp->pp_wpipe;  in pipe_paircreate()
387 	knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));  in pipe_paircreate()
396 	error = pipe_create(wpipe, false);  in pipe_paircreate()
412 	wpipe->pipe_state |= PIPE_DIRECTOK;  in pipe_paircreate()
417 	knlist_destroy(&wpipe->pipe_sel.si_note);  in pipe_paircreate()
477 	struct pipe *rpipe, *wpipe;  in kern_pipe()  local
485 	wpipe = &pp->pp_wpipe;  in kern_pipe()
489 		pipeclose(wpipe);  in kern_pipe()
511 		pipeclose(wpipe);  in kern_pipe()
515 	finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);  in kern_pipe()
943 pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio)  in pipe_build_write_buffer()  argument
948 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_build_write_buffer()
949 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_build_write_buffer()
950 	    ("%s: PIPE_DIRECTW set on %p", __func__, wpipe));  in pipe_build_write_buffer()
951 	KASSERT(wpipe->pipe_pages.cnt == 0,  in pipe_build_write_buffer()
952 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_build_write_buffer()
954 	if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)  in pipe_build_write_buffer()
955                 size = wpipe->pipe_buffer.size;  in pipe_build_write_buffer()
959 	wpipe->pipe_state |= PIPE_DIRECTW;  in pipe_build_write_buffer()
960 	PIPE_UNLOCK(wpipe);  in pipe_build_write_buffer()
963 	    wpipe->pipe_pages.ms, PIPENPAGES);  in pipe_build_write_buffer()
964 	PIPE_LOCK(wpipe);  in pipe_build_write_buffer()
966 		wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_build_write_buffer()
970 	wpipe->pipe_pages.npages = i;  in pipe_build_write_buffer()
971 	wpipe->pipe_pages.pos =  in pipe_build_write_buffer()
973 	wpipe->pipe_pages.cnt = size;  in pipe_build_write_buffer()
990 pipe_destroy_write_buffer(struct pipe *wpipe)  in pipe_destroy_write_buffer()  argument
993 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_destroy_write_buffer()
994 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_destroy_write_buffer()
995 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_destroy_write_buffer()
996 	KASSERT(wpipe->pipe_pages.cnt == 0,  in pipe_destroy_write_buffer()
997 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_destroy_write_buffer()
999 	wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_destroy_write_buffer()
1000 	vm_page_unhold_pages(wpipe->pipe_pages.ms, wpipe->pipe_pages.npages);  in pipe_destroy_write_buffer()
1001 	wpipe->pipe_pages.npages = 0;  in pipe_destroy_write_buffer()
1010 pipe_clone_write_buffer(struct pipe *wpipe)  in pipe_clone_write_buffer()  argument
1017 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_clone_write_buffer()
1018 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_clone_write_buffer()
1019 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_clone_write_buffer()
1021 	size = wpipe->pipe_pages.cnt;  in pipe_clone_write_buffer()
1022 	pos = wpipe->pipe_pages.pos;  in pipe_clone_write_buffer()
1023 	wpipe->pipe_pages.cnt = 0;  in pipe_clone_write_buffer()
1025 	wpipe->pipe_buffer.in = size;  in pipe_clone_write_buffer()
1026 	wpipe->pipe_buffer.out = 0;  in pipe_clone_write_buffer()
1027 	wpipe->pipe_buffer.cnt = size;  in pipe_clone_write_buffer()
1029 	PIPE_UNLOCK(wpipe);  in pipe_clone_write_buffer()
1030 	iov.iov_base = wpipe->pipe_buffer.buffer;  in pipe_clone_write_buffer()
1039 	uiomove_fromphys(wpipe->pipe_pages.ms, pos, size, &uio);  in pipe_clone_write_buffer()
1040 	PIPE_LOCK(wpipe);  in pipe_clone_write_buffer()
1041 	pipe_destroy_write_buffer(wpipe);  in pipe_clone_write_buffer()
1052 pipe_direct_write(struct pipe *wpipe, struct uio *uio)  in pipe_direct_write()  argument
1057 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_direct_write()
1058 	if ((wpipe->pipe_state & PIPE_EOF) != 0) {  in pipe_direct_write()
1062 	if (wpipe->pipe_state & PIPE_DIRECTW) {  in pipe_direct_write()
1063 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1064 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1065 			wakeup(wpipe);  in pipe_direct_write()
1067 		pipeselwakeup(wpipe);  in pipe_direct_write()
1068 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1069 		pipeunlock(wpipe);  in pipe_direct_write()
1070 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
1072 		pipelock(wpipe, 0);  in pipe_direct_write()
1077 	if (wpipe->pipe_buffer.cnt > 0) {  in pipe_direct_write()
1078 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1079 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1080 			wakeup(wpipe);  in pipe_direct_write()
1082 		pipeselwakeup(wpipe);  in pipe_direct_write()
1083 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1084 		pipeunlock(wpipe);  in pipe_direct_write()
1085 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
1087 		pipelock(wpipe, 0);  in pipe_direct_write()
1093 	error = pipe_build_write_buffer(wpipe, uio);  in pipe_direct_write()
1098 	while (wpipe->pipe_pages.cnt != 0 &&  in pipe_direct_write()
1099 	    (wpipe->pipe_state & PIPE_EOF) == 0) {  in pipe_direct_write()
1100 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1101 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1102 			wakeup(wpipe);  in pipe_direct_write()
1104 		pipeselwakeup(wpipe);  in pipe_direct_write()
1105 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1106 		pipeunlock(wpipe);  in pipe_direct_write()
1107 		error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,  in pipe_direct_write()
1109 		pipelock(wpipe, 0);  in pipe_direct_write()
1114 	if ((wpipe->pipe_state & PIPE_EOF) != 0) {  in pipe_direct_write()
1115 		wpipe->pipe_pages.cnt = 0;  in pipe_direct_write()
1116 		pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
1117 		pipeselwakeup(wpipe);  in pipe_direct_write()
1120 		pipe_clone_write_buffer(wpipe);  in pipe_direct_write()
1122 		pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
1124 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_direct_write()
1125 	    ("pipe %p leaked PIPE_DIRECTW", wpipe));  in pipe_direct_write()
1129 	wakeup(wpipe);  in pipe_direct_write()
1138 	struct pipe *wpipe, *rpipe;  in pipe_write()  local
1143 	wpipe = PIPE_PEER(rpipe);  in pipe_write()
1145 	error = pipelock(wpipe, 1);  in pipe_write()
1153 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_write()
1154 	    (wpipe->pipe_state & PIPE_EOF)) {  in pipe_write()
1155 		pipeunlock(wpipe);  in pipe_write()
1160 	error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);  in pipe_write()
1162 		pipeunlock(wpipe);  in pipe_write()
1167 	++wpipe->pipe_busy;  in pipe_write()
1170 	desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);  in pipe_write()
1171 	while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {  in pipe_write()
1183 	    wpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&  in pipe_write()
1184 	    wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&  in pipe_write()
1189 	if (desiredsize != wpipe->pipe_buffer.size &&  in pipe_write()
1190 	    (wpipe->pipe_state & PIPE_DIRECTW) == 0) {  in pipe_write()
1191 		PIPE_UNLOCK(wpipe);  in pipe_write()
1192 		pipespace(wpipe, desiredsize);  in pipe_write()
1193 		PIPE_LOCK(wpipe);  in pipe_write()
1195 	MPASS(wpipe->pipe_buffer.size != 0);  in pipe_write()
1202 		if (wpipe->pipe_state & PIPE_EOF) {  in pipe_write()
1218 		    wpipe->pipe_buffer.size >= pipe_mindirect &&  in pipe_write()
1220 			error = pipe_direct_write(wpipe, uio);  in pipe_write()
1234 		if (wpipe->pipe_pages.cnt != 0) {  in pipe_write()
1235 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1236 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1237 				wakeup(wpipe);  in pipe_write()
1239 			pipeselwakeup(wpipe);  in pipe_write()
1240 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1241 			pipeunlock(wpipe);  in pipe_write()
1242 			error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,  in pipe_write()
1244 			pipelock(wpipe, 0);  in pipe_write()
1250 		space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;  in pipe_write()
1275 			segsize = wpipe->pipe_buffer.size -  in pipe_write()
1276 				wpipe->pipe_buffer.in;  in pipe_write()
1283 			error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],  in pipe_write()
1288 				KASSERT(wpipe->pipe_buffer.in + segsize ==  in pipe_write()
1289 					wpipe->pipe_buffer.size,  in pipe_write()
1299 				    &wpipe->pipe_buffer.buffer[0],  in pipe_write()
1304 				wpipe->pipe_buffer.in += size;  in pipe_write()
1305 				if (wpipe->pipe_buffer.in >=  in pipe_write()
1306 				    wpipe->pipe_buffer.size) {  in pipe_write()
1307 					KASSERT(wpipe->pipe_buffer.in ==  in pipe_write()
1309 						wpipe->pipe_buffer.size,  in pipe_write()
1311 					wpipe->pipe_buffer.in = size - segsize;  in pipe_write()
1314 				wpipe->pipe_buffer.cnt += size;  in pipe_write()
1315 				KASSERT(wpipe->pipe_buffer.cnt <=  in pipe_write()
1316 					wpipe->pipe_buffer.size,  in pipe_write()
1326 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1327 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1328 				wakeup(wpipe);  in pipe_write()
1343 			pipeselwakeup(wpipe);  in pipe_write()
1345 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1346 			pipeunlock(wpipe);  in pipe_write()
1347 			error = msleep(wpipe, PIPE_MTX(rpipe),  in pipe_write()
1349 			pipelock(wpipe, 0);  in pipe_write()
1356 	--wpipe->pipe_busy;  in pipe_write()
1358 	if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {  in pipe_write()
1359 		wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);  in pipe_write()
1360 		wakeup(wpipe);  in pipe_write()
1361 	} else if (wpipe->pipe_buffer.cnt > 0) {  in pipe_write()
1366 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1367 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1368 			wakeup(wpipe);  in pipe_write()
1382 		pipe_timestamp(&wpipe->pipe_mtime);  in pipe_write()
1388 	if (wpipe->pipe_buffer.cnt)  in pipe_write()
1389 		pipeselwakeup(wpipe);  in pipe_write()
1391 	pipeunlock(wpipe);  in pipe_write()
1493 	struct pipe *wpipe;  in pipe_poll()  local
1501 	wpipe = PIPE_PEER(rpipe);  in pipe_poll()
1513 		if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1514 		    (wpipe->pipe_state & PIPE_EOF) ||  in pipe_poll()
1515 		    ((wpipe->pipe_state & PIPE_DIRECTW) == 0 &&  in pipe_poll()
1516 		     ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||  in pipe_poll()
1517 			 wpipe->pipe_buffer.size == 0)))  in pipe_poll()
1530 			if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1531 			    (wpipe->pipe_state & PIPE_EOF))  in pipe_poll()
1548 		    wpipe->pipe_present == PIPE_ACTIVE) {  in pipe_poll()
1549 			selrecord(td, &wpipe->pipe_sel);  in pipe_poll()
1550 			if (SEL_WAITING(&wpipe->pipe_sel))  in pipe_poll()
1551 				wpipe->pipe_state |= PIPE_SEL;  in pipe_poll()
1863 	struct pipe *wpipe = kn->kn_hook;  in filt_pipewrite()  local
1869 	if (wpipe->pipe_present == PIPE_ACTIVE ||  in filt_pipewrite()
1870 	    (wpipe->pipe_type & PIPE_TYPE_NAMED) != 0) {  in filt_pipewrite()
1871 		PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in filt_pipewrite()
1873 		if (wpipe->pipe_state & PIPE_DIRECTW) {  in filt_pipewrite()
1875 		} else if (wpipe->pipe_buffer.size > 0) {  in filt_pipewrite()
1876 			kn->kn_data = wpipe->pipe_buffer.size -  in filt_pipewrite()
1877 			    wpipe->pipe_buffer.cnt;  in filt_pipewrite()
1883 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in filt_pipewrite()
1884 	    (wpipe->pipe_state & PIPE_EOF)) {  in filt_pipewrite()