135 vmbus_chan_signal(const struct vmbus_channel *chan)  in vmbus_chan_signal()  argument
137 	atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask);  in vmbus_chan_signal()
138 	if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF)  in vmbus_chan_signal()
139 		atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask);  in vmbus_chan_signal()
142 		    (vm_offset_t)chan->ch_monprm));  in vmbus_chan_signal()
146 vmbus_chan_signal_tx(struct vmbus_channel *chan)  in vmbus_chan_signal_tx()  argument
148 	chan->ch_txbr.txbr_intrcnt ++;  in vmbus_chan_signal_tx()
150 	vmbus_chan_signal(chan);  in vmbus_chan_signal_tx()
154 vmbus_chan_signal_rx(struct vmbus_channel *chan)  in vmbus_chan_signal_rx()  argument
156 	chan->ch_rxbr.rxbr_intrcnt ++;  in vmbus_chan_signal_rx()
158 	vmbus_chan_signal(chan);  in vmbus_chan_signal_rx()
162 vmbus_chan_ins_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan)  in vmbus_chan_ins_prilist()  argument
166 	if (atomic_testandset_int(&chan->ch_stflags,  in vmbus_chan_ins_prilist()
169 	TAILQ_INSERT_TAIL(&sc->vmbus_prichans, chan, ch_prilink);  in vmbus_chan_ins_prilist()
173 vmbus_chan_rem_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan)  in vmbus_chan_rem_prilist()  argument
177 	if (atomic_testandclear_int(&chan->ch_stflags,  in vmbus_chan_rem_prilist()
180 	TAILQ_REMOVE(&sc->vmbus_prichans, chan, ch_prilink);  in vmbus_chan_rem_prilist()
185     struct vmbus_channel *chan)  in vmbus_chan_ins_sublist()  argument
190 	if (atomic_testandset_int(&chan->ch_stflags,  in vmbus_chan_ins_sublist()
193 	TAILQ_INSERT_TAIL(&prichan->ch_subchans, chan, ch_sublink);  in vmbus_chan_ins_sublist()
201     struct vmbus_channel *chan)  in vmbus_chan_rem_sublist()  argument
210 	if (atomic_testandclear_int(&chan->ch_stflags,  in vmbus_chan_rem_sublist()
213 	TAILQ_REMOVE(&prichan->ch_subchans, chan, ch_sublink);  in vmbus_chan_rem_sublist()
217 vmbus_chan_ins_list(struct vmbus_softc *sc, struct vmbus_channel *chan)  in vmbus_chan_ins_list()  argument
221 	if (atomic_testandset_int(&chan->ch_stflags,  in vmbus_chan_ins_list()
224 	TAILQ_INSERT_TAIL(&sc->vmbus_chans, chan, ch_link);  in vmbus_chan_ins_list()
228 vmbus_chan_rem_list(struct vmbus_softc *sc, struct vmbus_channel *chan)  in vmbus_chan_rem_list()  argument
232 	if (atomic_testandclear_int(&chan->ch_stflags,  in vmbus_chan_rem_list()
235 	TAILQ_REMOVE(&sc->vmbus_chans, chan, ch_link);  in vmbus_chan_rem_list()
241 	struct vmbus_channel *chan = arg1;  in vmbus_chan_sysctl_mnf()  local
244 	if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF)  in vmbus_chan_sysctl_mnf()
250 vmbus_chan_sysctl_create(struct vmbus_channel *chan)  in vmbus_chan_sysctl_create()  argument
263 	ctx = &chan->ch_sysctl_ctx;  in vmbus_chan_sysctl_create()
270 	    SYSCTL_CHILDREN(device_get_sysctl_tree(chan->ch_dev)),  in vmbus_chan_sysctl_create()
278 	if (VMBUS_CHAN_ISPRIMARY(chan))  in vmbus_chan_sysctl_create()
279 		ch_id = chan->ch_id;  in vmbus_chan_sysctl_create()
281 		ch_id = chan->ch_prichan->ch_id;  in vmbus_chan_sysctl_create()
288 	if (!VMBUS_CHAN_ISPRIMARY(chan)) {  in vmbus_chan_sysctl_create()
303 		snprintf(name, sizeof(name), "%d", chan->ch_subidx);  in vmbus_chan_sysctl_create()
310 		    "chanid", CTLFLAG_RD, &chan->ch_id, 0, "channel id");  in vmbus_chan_sysctl_create()
314 	    "cpu", CTLFLAG_RD, &chan->ch_cpuid, 0, "owner CPU id");  in vmbus_chan_sysctl_create()
317 	    chan, 0, vmbus_chan_sysctl_mnf, "I",  in vmbus_chan_sysctl_create()
326 		vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx");  in vmbus_chan_sysctl_create()
330 		vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx");  in vmbus_chan_sysctl_create()
335 vmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size,  in vmbus_chan_open()  argument
344 	KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated"));  in vmbus_chan_open()
345 	chan->ch_bufring_size = txbr_size + rxbr_size;  in vmbus_chan_open()
346 	chan->ch_bufring = contigmalloc(chan->ch_bufring_size, M_DEVBUF,  in vmbus_chan_open()
348 	if (chan->ch_bufring == NULL) {  in vmbus_chan_open()
349 		vmbus_chan_printf(chan, "bufring allocation failed\n");  in vmbus_chan_open()
353 	cbr.cbr = chan->ch_bufring;  in vmbus_chan_open()
354 	cbr.cbr_paddr = pmap_kextract((vm_offset_t)chan->ch_bufring);  in vmbus_chan_open()
358 	error = vmbus_chan_open_br(chan, &cbr, udata, udlen, cb, cbarg);  in vmbus_chan_open()
367 			vmbus_chan_printf(chan, "chan%u bufring GPADL "  in vmbus_chan_open()
369 			    "leak %d bytes memory\n", chan->ch_id,  in vmbus_chan_open()
372 			contigfree(chan->ch_bufring, chan->ch_bufring_size,  in vmbus_chan_open()
375 		chan->ch_bufring = NULL;  in vmbus_chan_open()
381 vmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr,  in vmbus_chan_open_br()  argument
384 	struct vmbus_softc *sc = chan->ch_vmbus;  in vmbus_chan_open_br()
394 		vmbus_chan_printf(chan,  in vmbus_chan_open_br()
395 		    "invalid udata len %d for chan%u\n", udlen, chan->ch_id);  in vmbus_chan_open_br()
414 	if (atomic_testandset_int(&chan->ch_stflags,  in vmbus_chan_open_br()
416 		panic("double-open chan%u", chan->ch_id);  in vmbus_chan_open_br()
418 	chan->ch_cb = cb;  in vmbus_chan_open_br()
419 	chan->ch_cbarg = cbarg;  in vmbus_chan_open_br()
421 	vmbus_chan_update_evtflagcnt(sc, chan);  in vmbus_chan_open_br()
423 	chan->ch_tq = VMBUS_PCPU_GET(chan->ch_vmbus, event_tq, chan->ch_cpuid);  in vmbus_chan_open_br()
424 	if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD)  in vmbus_chan_open_br()
428 	TASK_INIT(&chan->ch_task, 0, task_fn, chan);  in vmbus_chan_open_br()
431 	vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size);  in vmbus_chan_open_br()
433 	vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size);  in vmbus_chan_open_br()
436 	vmbus_chan_sysctl_create(chan);  in vmbus_chan_open_br()
441 	error = vmbus_chan_gpadl_connect(chan, cbr->cbr_paddr,  in vmbus_chan_open_br()
442 	    txbr_size + rxbr_size, &chan->ch_bufring_gpadl);  in vmbus_chan_open_br()
444 		vmbus_chan_printf(chan,  in vmbus_chan_open_br()
445 		    "failed to connect bufring GPADL to chan%u\n", chan->ch_id);  in vmbus_chan_open_br()
453 	vmbus_chan_set_chmap(chan);  in vmbus_chan_open_br()
460 		vmbus_chan_printf(chan,  in vmbus_chan_open_br()
462 		    chan->ch_id);  in vmbus_chan_open_br()
469 	req->chm_chanid = chan->ch_id;  in vmbus_chan_open_br()
470 	req->chm_openid = chan->ch_id;  in vmbus_chan_open_br()
471 	req->chm_gpadl = chan->ch_bufring_gpadl;  in vmbus_chan_open_br()
472 	req->chm_vcpuid = chan->ch_vcpuid;  in vmbus_chan_open_br()
479 		vmbus_chan_printf(chan,  in vmbus_chan_open_br()
481 		    chan->ch_id, error);  in vmbus_chan_open_br()
490 		if (vmbus_chan_is_revoked(chan)) {  in vmbus_chan_open_br()
498 			vmbus_chan_printf(chan,  in vmbus_chan_open_br()
500 			    chan->ch_id);  in vmbus_chan_open_br()
535 			vmbus_chan_printf(chan, "chan%u opened\n", chan->ch_id);  in vmbus_chan_open_br()
539 	vmbus_chan_printf(chan, "failed to open chan%u\n", chan->ch_id);  in vmbus_chan_open_br()
543 	sysctl_ctx_free(&chan->ch_sysctl_ctx);  in vmbus_chan_open_br()
544 	vmbus_chan_clear_chmap(chan);  in vmbus_chan_open_br()
545 	if (chan->ch_bufring_gpadl != 0) {  in vmbus_chan_open_br()
548 		error1 = vmbus_chan_gpadl_disconnect(chan,  in vmbus_chan_open_br()
549 		    chan->ch_bufring_gpadl);  in vmbus_chan_open_br()
557 		chan->ch_bufring_gpadl = 0;  in vmbus_chan_open_br()
559 	atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED);  in vmbus_chan_open_br()
564 vmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr,  in vmbus_chan_gpadl_connect()  argument
567 	struct vmbus_softc *sc = chan->ch_vmbus;  in vmbus_chan_gpadl_connect()
595 		vmbus_chan_printf(chan, "GPA too large, %d pages\n",  in vmbus_chan_gpadl_connect()
623 		vmbus_chan_printf(chan,  in vmbus_chan_gpadl_connect()
625 		    chan->ch_id);  in vmbus_chan_gpadl_connect()
631 	req->chm_chanid = chan->ch_id;  in vmbus_chan_gpadl_connect()
642 		vmbus_chan_printf(chan,  in vmbus_chan_gpadl_connect()
644 		    chan->ch_id, error);  in vmbus_chan_gpadl_connect()
679 		vmbus_chan_printf(chan, "gpadl_conn(chan%u) failed: %u\n",  in vmbus_chan_gpadl_connect()
680 		    chan->ch_id, status);  in vmbus_chan_gpadl_connect()
687 		vmbus_chan_printf(chan, "gpadl_conn(chan%u) succeeded\n",  in vmbus_chan_gpadl_connect()
688 		    chan->ch_id);  in vmbus_chan_gpadl_connect()
694 vmbus_chan_wait_revoke(const struct vmbus_channel *chan, bool can_sleep)  in vmbus_chan_wait_revoke()  argument
701 		if (vmbus_chan_is_revoked(chan))  in vmbus_chan_wait_revoke()
717 vmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl)  in vmbus_chan_gpadl_disconnect()  argument
719 	struct vmbus_softc *sc = chan->ch_vmbus;  in vmbus_chan_gpadl_disconnect()
728 		vmbus_chan_printf(chan,  in vmbus_chan_gpadl_disconnect()
730 		    chan->ch_id);  in vmbus_chan_gpadl_disconnect()
736 	req->chm_chanid = chan->ch_id;  in vmbus_chan_gpadl_disconnect()
743 		if (vmbus_chan_wait_revoke(chan, true)) {  in vmbus_chan_gpadl_disconnect()
748 			vmbus_chan_printf(chan,  in vmbus_chan_gpadl_disconnect()
750 			    "exec failed: %d\n", chan->ch_id, error);  in vmbus_chan_gpadl_disconnect()
753 		vmbus_chan_printf(chan,  in vmbus_chan_gpadl_disconnect()
755 		    chan->ch_id, error);  in vmbus_chan_gpadl_disconnect()
767 vmbus_chan_detach(struct vmbus_channel *chan)  in vmbus_chan_detach()  argument
771 	KASSERT(chan->ch_refs > 0, ("chan%u: invalid refcnt %d",  in vmbus_chan_detach()
772 	    chan->ch_id, chan->ch_refs));  in vmbus_chan_detach()
773 	refs = atomic_fetchadd_int(&chan->ch_refs, -1);  in vmbus_chan_detach()
775 	if (VMBUS_CHAN_ISPRIMARY(chan)) {  in vmbus_chan_detach()
777 		    chan->ch_id, refs + 1));  in vmbus_chan_detach()
785 			vmbus_chan_printf(chan, "chan%u detached\n",  in vmbus_chan_detach()
786 			    chan->ch_id);  in vmbus_chan_detach()
788 		taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task);  in vmbus_chan_detach()
795 	struct vmbus_channel *chan = xchan;  in vmbus_chan_clrchmap_task()  local
797 	chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL;  in vmbus_chan_clrchmap_task()
801 vmbus_chan_clear_chmap(struct vmbus_channel *chan)  in vmbus_chan_clear_chmap()  argument
805 	TASK_INIT(&chmap_task, 0, vmbus_chan_clrchmap_task, chan);  in vmbus_chan_clear_chmap()
806 	vmbus_chan_run_task(chan, &chmap_task);  in vmbus_chan_clear_chmap()
810 vmbus_chan_set_chmap(struct vmbus_channel *chan)  in vmbus_chan_set_chmap()  argument
813 	chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan;  in vmbus_chan_set_chmap()
824 vmbus_chan_poll_cancel(struct vmbus_channel *chan)  in vmbus_chan_poll_cancel()  argument
828 	TASK_INIT(&poll_cancel, 0, vmbus_chan_poll_cancel_task, chan);  in vmbus_chan_poll_cancel()
829 	vmbus_chan_run_task(chan, &poll_cancel);  in vmbus_chan_poll_cancel()
833 vmbus_chan_close_internal(struct vmbus_channel *chan)  in vmbus_chan_close_internal()  argument
835 	struct vmbus_softc *sc = chan->ch_vmbus;  in vmbus_chan_close_internal()
847 		old_stflags = chan->ch_stflags;  in vmbus_chan_close_internal()
848 		if (atomic_cmpset_int(&chan->ch_stflags, old_stflags,  in vmbus_chan_close_internal()
855 			vmbus_chan_printf(chan, "chan%u not opened\n",  in vmbus_chan_close_internal()
856 			    chan->ch_id);  in vmbus_chan_close_internal()
865 	sysctl_ctx_free(&chan->ch_sysctl_ctx);  in vmbus_chan_close_internal()
870 	vmbus_chan_poll_cancel(chan);  in vmbus_chan_close_internal()
878 	vmbus_chan_clear_chmap(chan);  in vmbus_chan_close_internal()
879 	taskqueue_drain(chan->ch_tq, &chan->ch_task);  in vmbus_chan_close_internal()
880 	chan->ch_tq = NULL;  in vmbus_chan_close_internal()
887 		vmbus_chan_printf(chan,  in vmbus_chan_close_internal()
889 		    chan->ch_id);  in vmbus_chan_close_internal()
896 	req->chm_chanid = chan->ch_id;  in vmbus_chan_close_internal()
902 		vmbus_chan_printf(chan,  in vmbus_chan_close_internal()
904 		    chan->ch_id, error);  in vmbus_chan_close_internal()
909 		vmbus_chan_printf(chan, "chan%u closed\n", chan->ch_id);  in vmbus_chan_close_internal()
915 	if (chan->ch_bufring_gpadl != 0) {  in vmbus_chan_close_internal()
918 		error1 = vmbus_chan_gpadl_disconnect(chan,  in vmbus_chan_close_internal()
919 		    chan->ch_bufring_gpadl);  in vmbus_chan_close_internal()
927 			vmbus_chan_printf(chan, "chan%u bufring GPADL "  in vmbus_chan_close_internal()
928 			    "is still connected after close\n", chan->ch_id);  in vmbus_chan_close_internal()
929 			chan->ch_bufring = NULL;  in vmbus_chan_close_internal()
936 		chan->ch_bufring_gpadl = 0;  in vmbus_chan_close_internal()
942 	if (chan->ch_bufring != NULL) {  in vmbus_chan_close_internal()
943 		contigfree(chan->ch_bufring, chan->ch_bufring_size, M_DEVBUF);  in vmbus_chan_close_internal()
944 		chan->ch_bufring = NULL;  in vmbus_chan_close_internal()
950 vmbus_chan_close_direct(struct vmbus_channel *chan)  in vmbus_chan_close_direct()  argument
955 	if (VMBUS_CHAN_ISPRIMARY(chan)) {  in vmbus_chan_close_direct()
962 		mtx_lock(&chan->ch_subchan_lock);  in vmbus_chan_close_direct()
963 		TAILQ_FOREACH(subchan, &chan->ch_subchans, ch_sublink) {  in vmbus_chan_close_direct()
967 			    chan->ch_id, subchan->ch_subidx));  in vmbus_chan_close_direct()
969 		mtx_unlock(&chan->ch_subchan_lock);  in vmbus_chan_close_direct()
973 	error = vmbus_chan_close_internal(chan);  in vmbus_chan_close_direct()
974 	if (!VMBUS_CHAN_ISPRIMARY(chan)) {  in vmbus_chan_close_direct()
979 		vmbus_chan_detach(chan);  in vmbus_chan_close_direct()
990 vmbus_chan_close(struct vmbus_channel *chan)  in vmbus_chan_close()  argument
994 	if (!VMBUS_CHAN_ISPRIMARY(chan)) {  in vmbus_chan_close()
1005 	subchan_cnt = chan->ch_subchan_cnt;  in vmbus_chan_close()
1010 		subchan = vmbus_subchan_get(chan, subchan_cnt);  in vmbus_chan_close()
1024 	vmbus_chan_close_internal(chan);  in vmbus_chan_close()
1028 vmbus_chan_intr_drain(struct vmbus_channel *chan)  in vmbus_chan_intr_drain()  argument
1031 	taskqueue_drain(chan->ch_tq, &chan->ch_task);  in vmbus_chan_intr_drain()
1035 vmbus_chan_write_available(struct vmbus_channel *chan)  in vmbus_chan_write_available()  argument
1037 	return (vmbus_txbr_available(&chan->ch_txbr));  in vmbus_chan_write_available()
1041 vmbus_chan_write_signal(struct vmbus_channel *chan,  in vmbus_chan_write_signal()  argument
1045 	    vmbus_chan_write_available(chan) > min_signal_size) {  in vmbus_chan_write_signal()
1049 	if (!vmbus_txbr_get_imask(&chan->ch_txbr)) {  in vmbus_chan_write_signal()
1051 		vmbus_chan_signal_tx(chan);  in vmbus_chan_write_signal()
1059 vmbus_chan_set_pending_send_size(struct vmbus_channel *chan,  in vmbus_chan_set_pending_send_size()  argument
1062 	if (chan)  in vmbus_chan_set_pending_send_size()
1063 		vmbus_txbr_set_pending_snd_sz(&chan->ch_txbr, size);  in vmbus_chan_set_pending_send_size()
1067 vmbus_chan_iov_send(struct vmbus_channel *chan,  in vmbus_chan_iov_send()  argument
1077 	error = vmbus_txbr_write_call(&chan->ch_txbr, iov, iovlen,  in vmbus_chan_iov_send()
1081 		vmbus_chan_signal_tx(chan);  in vmbus_chan_iov_send()
1088 vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags,  in vmbus_chan_send()  argument
1100 	KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr),  in vmbus_chan_send()
1116 	error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt);  in vmbus_chan_send()
1118 		vmbus_chan_signal_tx(chan);  in vmbus_chan_send()
1123 vmbus_chan_send_sglist(struct vmbus_channel *chan,  in vmbus_chan_send_sglist()  argument
1135 	KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr),  in vmbus_chan_send_sglist()
1155 	error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt);  in vmbus_chan_send_sglist()
1157 		vmbus_chan_signal_tx(chan);  in vmbus_chan_send_sglist()
1162 vmbus_chan_send_prplist(struct vmbus_channel *chan,  in vmbus_chan_send_prplist()  argument
1176 	KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr),  in vmbus_chan_send_prplist()
1196 	error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt);  in vmbus_chan_send_prplist()
1198 		vmbus_chan_signal_tx(chan);  in vmbus_chan_send_prplist()
1203 vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0,  in vmbus_chan_recv()  argument
1210 	error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt));  in vmbus_chan_recv()
1215 		vmbus_chan_printf(chan, "invalid hlen %u\n", pkt.cph_hlen);  in vmbus_chan_recv()
1220 		vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n",  in vmbus_chan_recv()
1239 	error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen, &sig_event);  in vmbus_chan_recv()
1243 		vmbus_chan_signal_rx(chan);  in vmbus_chan_recv()
1249 vmbus_chan_recv_pkt(struct vmbus_channel *chan,  in vmbus_chan_recv_pkt()  argument
1256 	error = vmbus_rxbr_peek(&chan->ch_rxbr, pkt, pkt_hlen);  in vmbus_chan_recv_pkt()
1261 		vmbus_chan_printf(chan, "invalid hlen %u\n", pkt->cph_hlen);  in vmbus_chan_recv_pkt()
1266 		vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n",  in vmbus_chan_recv_pkt()
1284 	error = vmbus_rxbr_read(&chan->ch_rxbr, pkt + 1,  in vmbus_chan_recv_pkt()
1289 		vmbus_chan_signal_rx(chan);  in vmbus_chan_recv_pkt()
1295 vmbus_chan_read_available(struct vmbus_channel *chan)  in vmbus_chan_read_available()  argument
1297 	return (vmbus_rxbr_available(&chan->ch_rxbr));  in vmbus_chan_read_available()
1309 vmbus_chan_recv_peek(struct vmbus_channel *chan,  in vmbus_chan_recv_peek()  argument
1318 	error = vmbus_rxbr_idxadv_peek(&chan->ch_rxbr,  in vmbus_chan_recv_peek()
1322 		vmbus_chan_signal_rx(chan);  in vmbus_chan_recv_peek()
1333 vmbus_chan_recv_idxadv(struct vmbus_channel *chan, uint32_t advance)  in vmbus_chan_recv_idxadv()  argument
1341 	error = vmbus_rxbr_idxadv(&chan->ch_rxbr, advance, &sig_event);  in vmbus_chan_recv_idxadv()
1344 		vmbus_chan_signal_rx(chan);  in vmbus_chan_recv_idxadv()
1356 vmbus_chan_recv_peek_call(struct vmbus_channel *chan, int data_len,  in vmbus_chan_recv_peek_call()  argument
1359 	if (!chan || data_len <= 0 || cb == NULL)  in vmbus_chan_recv_peek_call()
1362 	return (vmbus_rxbr_peek_call(&chan->ch_rxbr, data_len, skip,  in vmbus_chan_recv_peek_call()
1369 	struct vmbus_channel *chan = xchan;  in vmbus_chan_task()  local
1370 	vmbus_chan_callback_t cb = chan->ch_cb;  in vmbus_chan_task()
1371 	void *cbarg = chan->ch_cbarg;  in vmbus_chan_task()
1373 	KASSERT(chan->ch_poll_intvl == 0,  in vmbus_chan_task()
1374 	    ("chan%u: interrupted in polling mode", chan->ch_id));  in vmbus_chan_task()
1391 		cb(chan, cbarg);  in vmbus_chan_task()
1393 		left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr);  in vmbus_chan_task()
1398 		vmbus_rxbr_intr_mask(&chan->ch_rxbr);  in vmbus_chan_task()
1405 	struct vmbus_channel *chan = xchan;  in vmbus_chan_task_nobatch()  local
1407 	KASSERT(chan->ch_poll_intvl == 0,  in vmbus_chan_task_nobatch()
1408 	    ("chan%u: interrupted in polling mode", chan->ch_id));  in vmbus_chan_task_nobatch()
1409 	chan->ch_cb(chan, chan->ch_cbarg);  in vmbus_chan_task_nobatch()
1415 	struct vmbus_channel *chan = xchan;  in vmbus_chan_poll_timeout()  local
1417 	KASSERT(chan->ch_poll_intvl != 0,  in vmbus_chan_poll_timeout()
1418 	    ("chan%u: polling timeout in interrupt mode", chan->ch_id));  in vmbus_chan_poll_timeout()
1419 	taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task);  in vmbus_chan_poll_timeout()
1425 	struct vmbus_channel *chan = xchan;  in vmbus_chan_poll_task()  local
1427 	KASSERT(chan->ch_poll_intvl != 0,  in vmbus_chan_poll_task()
1428 	    ("chan%u: polling in interrupt mode", chan->ch_id));  in vmbus_chan_poll_task()
1429 	callout_reset_sbt_curcpu(&chan->ch_poll_timeo, chan->ch_poll_intvl, 0,  in vmbus_chan_poll_task()
1430 	    vmbus_chan_poll_timeout, chan, chan->ch_poll_flags);  in vmbus_chan_poll_task()
1431 	chan->ch_cb(chan, chan->ch_cbarg);  in vmbus_chan_poll_task()
1438 	struct vmbus_channel *chan = arg->poll_chan;  in vmbus_chan_pollcfg_task()  local
1448 	if (intvl == chan->ch_poll_intvl) {  in vmbus_chan_pollcfg_task()
1452 	chan->ch_poll_intvl = intvl;  in vmbus_chan_pollcfg_task()
1458 	chan->ch_poll_flags = poll_flags;  in vmbus_chan_pollcfg_task()
1469 	chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL;  in vmbus_chan_pollcfg_task()
1471 	vmbus_rxbr_intr_mask(&chan->ch_rxbr);  in vmbus_chan_pollcfg_task()
1478 	taskqueue_cancel(chan->ch_tq, &chan->ch_task, NULL);  in vmbus_chan_pollcfg_task()
1481 	taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task);  in vmbus_chan_pollcfg_task()
1485 vmbus_chan_poll_cancel_intq(struct vmbus_channel *chan)  in vmbus_chan_poll_cancel_intq()  argument
1488 	if (chan->ch_poll_intvl == 0) {  in vmbus_chan_poll_cancel_intq()
1497 	callout_drain(&chan->ch_poll_timeo);  in vmbus_chan_poll_cancel_intq()
1507 	chan->ch_poll_intvl = 0;  in vmbus_chan_poll_cancel_intq()
1515 	taskqueue_cancel(chan->ch_tq, &chan->ch_poll_task, NULL);  in vmbus_chan_poll_cancel_intq()
1524 	struct vmbus_channel *chan = xchan;  in vmbus_chan_polldis_task()  local
1526 	if (!vmbus_chan_poll_cancel_intq(chan)) {  in vmbus_chan_polldis_task()
1535 	chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan;  in vmbus_chan_polldis_task()
1537 	vmbus_rxbr_intr_unmask(&chan->ch_rxbr);  in vmbus_chan_polldis_task()
1543 	taskqueue_enqueue(chan->ch_tq, &chan->ch_task);  in vmbus_chan_polldis_task()
1564 			struct vmbus_channel *chan;  in vmbus_event_flags_proc()  local
1569 			chan = sc->vmbus_chmap[chid_base + chid_ofs];  in vmbus_event_flags_proc()
1570 			if (__predict_false(chan == NULL)) {  in vmbus_event_flags_proc()
1576 			if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD)  in vmbus_event_flags_proc()
1577 				vmbus_rxbr_intr_mask(&chan->ch_rxbr);  in vmbus_event_flags_proc()
1578 			taskqueue_enqueue(chan->ch_tq, &chan->ch_task);  in vmbus_event_flags_proc()
1611     const struct vmbus_channel *chan)  in vmbus_chan_update_evtflagcnt()  argument
1616 	flag_cnt = (chan->ch_id / VMBUS_EVTFLAG_LEN) + 1;  in vmbus_chan_update_evtflagcnt()
1617 	flag_cnt_ptr = VMBUS_PCPU_PTR(sc, event_flags_cnt, chan->ch_cpuid);  in vmbus_chan_update_evtflagcnt()
1627 				vmbus_chan_printf(chan,  in vmbus_chan_update_evtflagcnt()
1629 				    chan->ch_id, chan->ch_cpuid, flag_cnt);  in vmbus_chan_update_evtflagcnt()
1639 	struct vmbus_channel *chan;  in vmbus_chan_alloc()  local
1641 	chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO);  in vmbus_chan_alloc()
1643 	chan->ch_monprm = contigmalloc(sizeof(struct hyperv_mon_param),  in vmbus_chan_alloc()
1645 	if (chan->ch_monprm == NULL) {  in vmbus_chan_alloc()
1647 		free(chan, M_DEVBUF);  in vmbus_chan_alloc()
1651 	chan->ch_refs = 1;  in vmbus_chan_alloc()
1652 	chan->ch_vmbus = sc;  in vmbus_chan_alloc()
1653 	mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF);  in vmbus_chan_alloc()
1654 	sx_init(&chan->ch_orphan_lock, "vmbus chorphan");  in vmbus_chan_alloc()
1655 	TAILQ_INIT(&chan->ch_subchans);  in vmbus_chan_alloc()
1656 	vmbus_rxbr_init(&chan->ch_rxbr);  in vmbus_chan_alloc()
1657 	vmbus_txbr_init(&chan->ch_txbr);  in vmbus_chan_alloc()
1659 	TASK_INIT(&chan->ch_poll_task, 0, vmbus_chan_poll_task, chan);  in vmbus_chan_alloc()
1660 	callout_init(&chan->ch_poll_timeo, 1);  in vmbus_chan_alloc()
1662 	return chan;  in vmbus_chan_alloc()
1666 vmbus_chan_free(struct vmbus_channel *chan)  in vmbus_chan_free()  argument
1669 	KASSERT(TAILQ_EMPTY(&chan->ch_subchans) && chan->ch_subchan_cnt == 0,  in vmbus_chan_free()
1671 	KASSERT((chan->ch_stflags &  in vmbus_chan_free()
1676 	KASSERT(chan->ch_orphan_xact == NULL,  in vmbus_chan_free()
1678 	KASSERT(chan->ch_refs == 0, ("chan%u: invalid refcnt %d",  in vmbus_chan_free()
1679 	    chan->ch_id, chan->ch_refs));  in vmbus_chan_free()
1680 	KASSERT(chan->ch_poll_intvl == 0, ("chan%u: polling is activated",  in vmbus_chan_free()
1681 	    chan->ch_id));  in vmbus_chan_free()
1683 	contigfree(chan->ch_monprm, sizeof(struct hyperv_mon_param), M_DEVBUF);  in vmbus_chan_free()
1684 	mtx_destroy(&chan->ch_subchan_lock);  in vmbus_chan_free()
1685 	sx_destroy(&chan->ch_orphan_lock);  in vmbus_chan_free()
1686 	vmbus_rxbr_deinit(&chan->ch_rxbr);  in vmbus_chan_free()
1687 	vmbus_txbr_deinit(&chan->ch_txbr);  in vmbus_chan_free()
1688 	free(chan, M_DEVBUF);  in vmbus_chan_free()
1797 vmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu)  in vmbus_chan_cpu_set()  argument
1801 	if (chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WS2008 ||  in vmbus_chan_cpu_set()
1802 	    chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WIN7) {  in vmbus_chan_cpu_set()
1807 	chan->ch_cpuid = cpu;  in vmbus_chan_cpu_set()
1808 	chan->ch_vcpuid = VMBUS_PCPU_GET(chan->ch_vmbus, vcpuid, cpu);  in vmbus_chan_cpu_set()
1811 		vmbus_chan_printf(chan,  in vmbus_chan_cpu_set()
1813 		    chan->ch_id, chan->ch_cpuid, chan->ch_vcpuid);  in vmbus_chan_cpu_set()
1818 vmbus_chan_cpu_rr(struct vmbus_channel *chan)  in vmbus_chan_cpu_rr()  argument
1824 	vmbus_chan_cpu_set(chan, cpu);  in vmbus_chan_cpu_rr()
1828 vmbus_chan_cpu_default(struct vmbus_channel *chan)  in vmbus_chan_cpu_default()  argument
1835 	vmbus_chan_cpu_set(chan, 0);  in vmbus_chan_cpu_default()
1843 	struct vmbus_channel *chan;  in vmbus_chan_msgproc_choffer()  local
1849 	chan = vmbus_chan_alloc(sc);  in vmbus_chan_msgproc_choffer()
1850 	if (chan == NULL) {  in vmbus_chan_msgproc_choffer()
1856 	chan->ch_id = offer->chm_chanid;  in vmbus_chan_msgproc_choffer()
1857 	chan->ch_subidx = offer->chm_subidx;  in vmbus_chan_msgproc_choffer()
1858 	chan->ch_guid_type = offer->chm_chtype;  in vmbus_chan_msgproc_choffer()
1859 	chan->ch_guid_inst = offer->chm_chinst;  in vmbus_chan_msgproc_choffer()
1862 	chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD;  in vmbus_chan_msgproc_choffer()
1864 	chan->ch_monprm->mp_connid = VMBUS_CONNID_EVENT;  in vmbus_chan_msgproc_choffer()
1866 		chan->ch_monprm->mp_connid = offer->chm_connid;  in vmbus_chan_msgproc_choffer()
1874 		chan->ch_txflags |= VMBUS_CHAN_TXF_HASMNF;  in vmbus_chan_msgproc_choffer()
1879 		chan->ch_montrig =  in vmbus_chan_msgproc_choffer()
1882 		chan->ch_montrig_mask =  in vmbus_chan_msgproc_choffer()
1888 		chan->ch_is_hvs = true;  in vmbus_chan_msgproc_choffer()
1890 		chan->ch_hvs_conn_from_host =  in vmbus_chan_msgproc_choffer()
1897 			    chan->ch_id,  in vmbus_chan_msgproc_choffer()
1898 			    (chan->ch_hvs_conn_from_host != 0) ?  in vmbus_chan_msgproc_choffer()
1902 		chan->ch_is_hvs = false;  in vmbus_chan_msgproc_choffer()
1908 	chan->ch_evtflag =  in vmbus_chan_msgproc_choffer()
1909 	    &sc->vmbus_tx_evtflags[chan->ch_id >> VMBUS_EVTFLAG_SHIFT];  in vmbus_chan_msgproc_choffer()
1910 	chan->ch_evtflag_mask = 1UL << (chan->ch_id & VMBUS_EVTFLAG_MASK);  in vmbus_chan_msgproc_choffer()
1915 	if (VMBUS_CHAN_ISPRIMARY(chan)) {  in vmbus_chan_msgproc_choffer()
1916 		chan->ch_mgmt_tq = sc->vmbus_devtq;  in vmbus_chan_msgproc_choffer()
1920 		chan->ch_mgmt_tq = sc->vmbus_subchtq;  in vmbus_chan_msgproc_choffer()
1924 	TASK_INIT(&chan->ch_attach_task, 0, attach_fn, chan);  in vmbus_chan_msgproc_choffer()
1925 	TASK_INIT(&chan->ch_detach_task, 0, detach_fn, chan);  in vmbus_chan_msgproc_choffer()
1927 	error = vmbus_chan_add(chan);  in vmbus_chan_msgproc_choffer()
1930 		    chan->ch_id, error);  in vmbus_chan_msgproc_choffer()
1931 		atomic_subtract_int(&chan->ch_refs, 1);  in vmbus_chan_msgproc_choffer()
1932 		vmbus_chan_free(chan);  in vmbus_chan_msgproc_choffer()
1935 	taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_attach_task);  in vmbus_chan_msgproc_choffer()
1943 	struct vmbus_channel *chan;  in vmbus_chan_msgproc_chrescind()  local
1956 	TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) {  in vmbus_chan_msgproc_chrescind()
1957 		if (chan->ch_id == note->chm_chanid)  in vmbus_chan_msgproc_chrescind()
1960 	if (chan == NULL) {  in vmbus_chan_msgproc_chrescind()
1966 	vmbus_chan_rem_list(sc, chan);  in vmbus_chan_msgproc_chrescind()
1969 	if (VMBUS_CHAN_ISPRIMARY(chan)) {  in vmbus_chan_msgproc_chrescind()
1978 		vmbus_chan_rem_prilist(sc, chan);  in vmbus_chan_msgproc_chrescind()
1988 	if (atomic_testandset_int(&chan->ch_stflags,  in vmbus_chan_msgproc_chrescind()
1992 	sx_xlock(&chan->ch_orphan_lock);  in vmbus_chan_msgproc_chrescind()
1993 	if (chan->ch_orphan_xact != NULL)  in vmbus_chan_msgproc_chrescind()
1994 		vmbus_xact_ctx_orphan(chan->ch_orphan_xact);  in vmbus_chan_msgproc_chrescind()
1995 	sx_xunlock(&chan->ch_orphan_lock);  in vmbus_chan_msgproc_chrescind()
1998 		vmbus_chan_printf(chan, "chan%u revoked\n", note->chm_chanid);  in vmbus_chan_msgproc_chrescind()
1999 	vmbus_chan_detach(chan);  in vmbus_chan_msgproc_chrescind()
2003 vmbus_chan_release(struct vmbus_channel *chan)  in vmbus_chan_release()  argument
2005 	struct vmbus_softc *sc = chan->ch_vmbus;  in vmbus_chan_release()
2012 		vmbus_chan_printf(chan,  in vmbus_chan_release()
2014 		    chan->ch_id);  in vmbus_chan_release()
2020 	req->chm_chanid = chan->ch_id;  in vmbus_chan_release()
2026 		vmbus_chan_printf(chan,  in vmbus_chan_release()
2028 		    chan->ch_id, error);  in vmbus_chan_release()
2031 			vmbus_chan_printf(chan, "chan%u freed\n", chan->ch_id);  in vmbus_chan_release()
2039 	struct vmbus_channel *chan = xchan;  in vmbus_prichan_detach_task()  local
2041 	KASSERT(VMBUS_CHAN_ISPRIMARY(chan),  in vmbus_prichan_detach_task()
2042 	    ("chan%u is not primary channel", chan->ch_id));  in vmbus_prichan_detach_task()
2045 	vmbus_delete_child(chan);  in vmbus_prichan_detach_task()
2048 	vmbus_chan_release(chan);  in vmbus_prichan_detach_task()
2051 	vmbus_chan_free(chan);  in vmbus_prichan_detach_task()
2057 	struct vmbus_channel *chan = xchan;  in vmbus_subchan_detach_task()  local
2058 	struct vmbus_channel *pri_chan = chan->ch_prichan;  in vmbus_subchan_detach_task()
2060 	KASSERT(!VMBUS_CHAN_ISPRIMARY(chan),  in vmbus_subchan_detach_task()
2061 	    ("chan%u is primary channel", chan->ch_id));  in vmbus_subchan_detach_task()
2064 	vmbus_chan_release(chan);  in vmbus_subchan_detach_task()
2068 	vmbus_chan_rem_sublist(pri_chan, chan);  in vmbus_subchan_detach_task()
2074 	vmbus_chan_free(chan);  in vmbus_subchan_detach_task()
2103 		struct vmbus_channel *chan;  in vmbus_chan_destroy_all()  local
2106 		TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) {  in vmbus_chan_destroy_all()
2107 			if (VMBUS_CHAN_ISPRIMARY(chan))  in vmbus_chan_destroy_all()
2110 		if (chan == NULL) {  in vmbus_chan_destroy_all()
2115 		vmbus_chan_rem_list(sc, chan);  in vmbus_chan_destroy_all()
2119 		vmbus_chan_rem_prilist(sc, chan);  in vmbus_chan_destroy_all()
2122 		taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task);  in vmbus_chan_destroy_all()
2129 	struct vmbus_channel **ret, *chan;  in vmbus_subchan_get()  local
2143 	TAILQ_FOREACH(chan, &pri_chan->ch_subchans, ch_sublink) {  in vmbus_subchan_get()
2145 		ret[i] = chan;  in vmbus_subchan_get()
2191 vmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on)  in vmbus_chan_set_readbatch()  argument
2194 		chan->ch_flags &= ~VMBUS_CHAN_FLAG_BATCHREAD;  in vmbus_chan_set_readbatch()
2196 		chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD;  in vmbus_chan_set_readbatch()
2200 vmbus_chan_id(const struct vmbus_channel *chan)  in vmbus_chan_id()  argument
2202 	return chan->ch_id;  in vmbus_chan_id()
2206 vmbus_chan_subidx(const struct vmbus_channel *chan)  in vmbus_chan_subidx()  argument
2208 	return chan->ch_subidx;  in vmbus_chan_subidx()
2212 vmbus_chan_is_primary(const struct vmbus_channel *chan)  in vmbus_chan_is_primary()  argument
2214 	if (VMBUS_CHAN_ISPRIMARY(chan))  in vmbus_chan_is_primary()
2221 vmbus_chan_is_hvs(const struct vmbus_channel *chan)  in vmbus_chan_is_hvs()  argument
2223 	return chan->ch_is_hvs;  in vmbus_chan_is_hvs()
2227 vmbus_chan_is_hvs_conn_from_host(const struct vmbus_channel *chan)  in vmbus_chan_is_hvs_conn_from_host()  argument
2229 	KASSERT(vmbus_chan_is_hvs(chan) == true,  in vmbus_chan_is_hvs_conn_from_host()
2230 	    ("Not a HyperV Socket channel %u", chan->ch_id));  in vmbus_chan_is_hvs_conn_from_host()
2231 	if (chan->ch_hvs_conn_from_host != 0)  in vmbus_chan_is_hvs_conn_from_host()
2238 vmbus_chan_guid_type(struct vmbus_channel *chan)  in vmbus_chan_guid_type()  argument
2240 	return &chan->ch_guid_type;  in vmbus_chan_guid_type()
2244 vmbus_chan_guid_inst(struct vmbus_channel *chan)  in vmbus_chan_guid_inst()  argument
2246 	return &chan->ch_guid_inst;  in vmbus_chan_guid_inst()
2263 vmbus_chan_tx_empty(const struct vmbus_channel *chan)  in vmbus_chan_tx_empty()  argument
2266 	return (vmbus_txbr_empty(&chan->ch_txbr));  in vmbus_chan_tx_empty()
2270 vmbus_chan_rx_empty(const struct vmbus_channel *chan)  in vmbus_chan_rx_empty()  argument
2273 	return (vmbus_rxbr_empty(&chan->ch_rxbr));  in vmbus_chan_rx_empty()
2277 vmbus_chan_printf(const struct vmbus_channel *chan, const char *fmt, ...)  in vmbus_chan_printf()  argument
2283 	if (chan->ch_dev == NULL || !device_is_alive(chan->ch_dev))  in vmbus_chan_printf()
2284 		dev = chan->ch_vmbus->vmbus_dev;  in vmbus_chan_printf()
2286 		dev = chan->ch_dev;  in vmbus_chan_printf()
2297 vmbus_chan_run_task(struct vmbus_channel *chan, struct task *task)  in vmbus_chan_run_task()  argument
2300 	taskqueue_enqueue(chan->ch_tq, task);  in vmbus_chan_run_task()
2301 	taskqueue_drain(chan->ch_tq, task);  in vmbus_chan_run_task()
2305 vmbus_chan_mgmt_tq(const struct vmbus_channel *chan)  in vmbus_chan_mgmt_tq()  argument
2308 	return (chan->ch_mgmt_tq);  in vmbus_chan_mgmt_tq()
2312 vmbus_chan_is_revoked(const struct vmbus_channel *chan)  in vmbus_chan_is_revoked()  argument
2315 	if (chan->ch_stflags & VMBUS_CHAN_ST_REVOKED)  in vmbus_chan_is_revoked()
2321 vmbus_chan_set_orphan(struct vmbus_channel *chan, struct vmbus_xact_ctx *xact)  in vmbus_chan_set_orphan()  argument
2324 	sx_xlock(&chan->ch_orphan_lock);  in vmbus_chan_set_orphan()
2325 	chan->ch_orphan_xact = xact;  in vmbus_chan_set_orphan()
2326 	sx_xunlock(&chan->ch_orphan_lock);  in vmbus_chan_set_orphan()
2330 vmbus_chan_unset_orphan(struct vmbus_channel *chan)  in vmbus_chan_unset_orphan()  argument
2333 	sx_xlock(&chan->ch_orphan_lock);  in vmbus_chan_unset_orphan()
2334 	chan->ch_orphan_xact = NULL;  in vmbus_chan_unset_orphan()
2335 	sx_xunlock(&chan->ch_orphan_lock);  in vmbus_chan_unset_orphan()
2339 vmbus_chan_xact_wait(const struct vmbus_channel *chan,  in vmbus_chan_xact_wait()  argument
2348 	if (vmbus_chan_is_revoked(chan)) {  in vmbus_chan_xact_wait()
2362 		while (!vmbus_chan_rx_empty(chan)) {  in vmbus_chan_xact_wait()
2373 vmbus_chan_poll_enable(struct vmbus_channel *chan, u_int pollhz)  in vmbus_chan_poll_enable()  argument
2378 	KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD,  in vmbus_chan_poll_enable()
2379 	    ("enable polling on non-batch chan%u", chan->ch_id));  in vmbus_chan_poll_enable()
2383 	arg.poll_chan = chan;  in vmbus_chan_poll_enable()
2386 	vmbus_chan_run_task(chan, &poll_cfg);  in vmbus_chan_poll_enable()
2390 vmbus_chan_poll_disable(struct vmbus_channel *chan)  in vmbus_chan_poll_disable()  argument
2394 	KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD,  in vmbus_chan_poll_disable()
2395 	    ("disable polling on non-batch chan%u", chan->ch_id));  in vmbus_chan_poll_disable()
2397 	TASK_INIT(&poll_dis, 0, vmbus_chan_polldis_task, chan);  in vmbus_chan_poll_disable()
2398 	vmbus_chan_run_task(chan, &poll_dis);  in vmbus_chan_poll_disable()