127 alloc_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int intr_idx,  in alloc_nm_rxq()  argument
142 	rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map,  in alloc_nm_rxq()
143 	    &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc);  in alloc_nm_rxq()
148 	rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map,  in alloc_nm_rxq()
149 	    &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc);  in alloc_nm_rxq()
153 	nm_rxq->vi = vi;  in alloc_nm_rxq()
154 	nm_rxq->nid = idx;  in alloc_nm_rxq()
155 	nm_rxq->iq_cidx = 0;  in alloc_nm_rxq()
156 	nm_rxq->iq_sidx = vi->qsize_rxq - sc->params.sge.spg_len / IQ_ESIZE;  in alloc_nm_rxq()
157 	nm_rxq->iq_gen = F_RSPD_GEN;  in alloc_nm_rxq()
158 	nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;  in alloc_nm_rxq()
159 	nm_rxq->fl_sidx = na->num_rx_desc;  in alloc_nm_rxq()
160 	nm_rxq->fl_sidx2 = nm_rxq->fl_sidx;	/* copy for rxsync cacheline */  in alloc_nm_rxq()
161 	nm_rxq->intr_idx = intr_idx;  in alloc_nm_rxq()
162 	nm_rxq->iq_cntxt_id = INVALID_NM_RXQ_CNTXT_ID;  in alloc_nm_rxq()
173 	    &nm_rxq->iq_abs_id, 0, "absolute id of the queue");  in alloc_nm_rxq()
175 	    &nm_rxq->iq_cntxt_id, 0, "SGE context id of the queue");  in alloc_nm_rxq()
177 	    &nm_rxq->iq_cidx, 0, "consumer index");  in alloc_nm_rxq()
185 	    &nm_rxq->fl_cntxt_id, 0, "SGE context id of the freelist");  in alloc_nm_rxq()
187 	    &nm_rxq->fl_cidx, 0, "consumer index");  in alloc_nm_rxq()
189 	    &nm_rxq->fl_pidx, 0, "producer index");  in alloc_nm_rxq()
195 free_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq)  in free_nm_rxq()  argument
202 	if (nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID)  in free_nm_rxq()
203 		free_nm_rxq_hwq(vi, nm_rxq);  in free_nm_rxq()
204 	MPASS(nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID);  in free_nm_rxq()
206 	free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba,  in free_nm_rxq()
207 	    nm_rxq->iq_desc);  in free_nm_rxq()
208 	free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba,  in free_nm_rxq()
209 	    nm_rxq->fl_desc);  in free_nm_rxq()
279 alloc_nm_rxq_hwq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int cong)  in alloc_nm_rxq_hwq()  argument
289 	MPASS(nm_rxq->iq_desc != NULL);  in alloc_nm_rxq_hwq()
290 	MPASS(nm_rxq->fl_desc != NULL);  in alloc_nm_rxq_hwq()
292 	bzero(nm_rxq->iq_desc, vi->qsize_rxq * IQ_ESIZE);  in alloc_nm_rxq_hwq()
293 	bzero(nm_rxq->fl_desc, na->num_rx_desc * EQ_ESIZE + sp->spg_len);  in alloc_nm_rxq_hwq()
300 	if (nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID)  in alloc_nm_rxq_hwq()
303 		c.iqid = htobe16(nm_rxq->iq_cntxt_id);  in alloc_nm_rxq_hwq()
304 		c.fl0id = htobe16(nm_rxq->fl_cntxt_id);  in alloc_nm_rxq_hwq()
306 		c.physiqid = htobe16(nm_rxq->iq_abs_id);  in alloc_nm_rxq_hwq()
309 	KASSERT(nm_rxq->intr_idx < sc->intr_count,  in alloc_nm_rxq_hwq()
310 	    ("%s: invalid direct intr_idx %d", __func__, nm_rxq->intr_idx));  in alloc_nm_rxq_hwq()
311 	v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx);  in alloc_nm_rxq_hwq()
321 	c.iqaddr = htobe64(nm_rxq->iq_ba);  in alloc_nm_rxq_hwq()
338 	c.fl0addr = htobe64(nm_rxq->fl_ba);  in alloc_nm_rxq_hwq()
347 	nm_rxq->iq_cidx = 0;  in alloc_nm_rxq_hwq()
348 	MPASS(nm_rxq->iq_sidx == vi->qsize_rxq - sp->spg_len / IQ_ESIZE);  in alloc_nm_rxq_hwq()
349 	nm_rxq->iq_gen = F_RSPD_GEN;  in alloc_nm_rxq_hwq()
350 	nm_rxq->iq_cntxt_id = be16toh(c.iqid);  in alloc_nm_rxq_hwq()
351 	nm_rxq->iq_abs_id = be16toh(c.physiqid);  in alloc_nm_rxq_hwq()
352 	cntxt_id = nm_rxq->iq_cntxt_id - sc->sge.iq_start;  in alloc_nm_rxq_hwq()
357 	sc->sge.iqmap[cntxt_id] = (void *)nm_rxq;  in alloc_nm_rxq_hwq()
359 	nm_rxq->fl_cntxt_id = be16toh(c.fl0id);  in alloc_nm_rxq_hwq()
360 	nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;  in alloc_nm_rxq_hwq()
361 	nm_rxq->fl_db_saved = 0;  in alloc_nm_rxq_hwq()
363 	nm_rxq->fl_db_threshold = chip_id(sc) <= CHELSIO_T5 ? 8 : 4;  in alloc_nm_rxq_hwq()
364 	MPASS(nm_rxq->fl_sidx == na->num_rx_desc);  in alloc_nm_rxq_hwq()
365 	cntxt_id = nm_rxq->fl_cntxt_id - sc->sge.eq_start;  in alloc_nm_rxq_hwq()
370 	sc->sge.eqmap[cntxt_id] = (void *)nm_rxq;  in alloc_nm_rxq_hwq()
372 	nm_rxq->fl_db_val = V_QID(nm_rxq->fl_cntxt_id) |  in alloc_nm_rxq_hwq()
380 		    V_FW_PARAMS_PARAM_YZ(nm_rxq->iq_cntxt_id);  in alloc_nm_rxq_hwq()
396 			    "ingress queue %d: %d\n", nm_rxq->iq_cntxt_id, rc);  in alloc_nm_rxq_hwq()
401 	    V_INGRESSQID(nm_rxq->iq_cntxt_id) |  in alloc_nm_rxq_hwq()
408 free_nm_rxq_hwq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq)  in free_nm_rxq_hwq()  argument
414 	    nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, 0xffff);  in free_nm_rxq_hwq()
417 		    __func__, nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, rc);  in free_nm_rxq_hwq()
418 	nm_rxq->iq_cntxt_id = INVALID_NM_RXQ_CNTXT_ID;  in free_nm_rxq_hwq()
451 		V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));  in alloc_nm_txq_hwq()
535 	struct sge_nm_rxq *nm_rxq;  in cxgbe_netmap_simple_rss()  local
544 	for_each_nm_rxq(vi, j, nm_rxq) {  in cxgbe_netmap_simple_rss()
545 		nm_state = atomic_load_int(&nm_rxq->nm_state);  in cxgbe_netmap_simple_rss()
546 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_simple_rss()
549 			MPASS(nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_simple_rss()
551 				defq = nm_rxq->iq_abs_id;  in cxgbe_netmap_simple_rss()
563 			for_each_nm_rxq(vi, j, nm_rxq) {  in cxgbe_netmap_simple_rss()
564 				nm_state = atomic_load_int(&nm_rxq->nm_state);  in cxgbe_netmap_simple_rss()
565 				kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_simple_rss()
570 					MPASS(nm_rxq->iq_cntxt_id !=  in cxgbe_netmap_simple_rss()
572 					vi->nm_rss[i++] = nm_rxq->iq_abs_id;  in cxgbe_netmap_simple_rss()
603 	struct sge_nm_rxq *nm_rxq;  in cxgbe_netmap_split_rss()  local
612 	for_each_nm_rxq(vi, i, nm_rxq) {  in cxgbe_netmap_split_rss()
614 		nm_state = atomic_load_int(&nm_rxq->nm_state);  in cxgbe_netmap_split_rss()
615 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_split_rss()
618 			MPASS(nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_split_rss()
621 				dq[j] = nm_rxq->iq_abs_id;  in cxgbe_netmap_split_rss()
643 	nm_rxq = &sc->sge.nm_rxq[vi->first_nm_rxq];  in cxgbe_netmap_split_rss()
646 			nm_state = atomic_load_int(&nm_rxq[j].nm_state);  in cxgbe_netmap_split_rss()
647 			kring = na->rx_rings[nm_rxq[j].nid];  in cxgbe_netmap_split_rss()
654 			MPASS(nm_rxq[j].iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_split_rss()
655 			if (dq_norss && defq == nm_rxq[j].iq_abs_id)  in cxgbe_netmap_split_rss()
657 			vi->nm_rss[i++] = nm_rxq[j].iq_abs_id;  in cxgbe_netmap_split_rss()
664 			nm_state = atomic_load_int(&nm_rxq[j].nm_state);  in cxgbe_netmap_split_rss()
665 			kring = na->rx_rings[nm_rxq[j].nid];  in cxgbe_netmap_split_rss()
672 			MPASS(nm_rxq[j].iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_split_rss()
673 			if (dq_norss && defq == nm_rxq[j].iq_abs_id)  in cxgbe_netmap_split_rss()
675 			vi->nm_rss[i++] = nm_rxq[j].iq_abs_id;  in cxgbe_netmap_split_rss()
710 	struct sge_nm_rxq *nm_rxq;  in cxgbe_netmap_on()  local
746 	for_each_nm_rxq(vi, i, nm_rxq) {  in cxgbe_netmap_on()
747 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_on()
751 		alloc_nm_rxq_hwq(vi, nm_rxq, tnl_cong(vi->pi, nm_cong_drop));  in cxgbe_netmap_on()
752 		nm_rxq->fl_hwidx = hwidx;  in cxgbe_netmap_on()
758 		MPASS(na->num_rx_desc == nm_rxq->fl_sidx);  in cxgbe_netmap_on()
759 		for (j = 0; j < nm_rxq->fl_sidx; j++) {  in cxgbe_netmap_on()
764 			nm_rxq->fl_desc[j] = htobe64(ba | hwidx);  in cxgbe_netmap_on()
766 		j = nm_rxq->fl_pidx = nm_rxq->fl_sidx - 8;  in cxgbe_netmap_on()
771 		    nm_rxq->fl_db_val | V_PIDX(j));  in cxgbe_netmap_on()
773 		(void) atomic_cmpset_int(&nm_rxq->nm_state, NM_OFF, NM_ON);  in cxgbe_netmap_on()
801 	struct sge_nm_rxq *nm_rxq;  in cxgbe_netmap_off()  local
836 	for_each_nm_rxq(vi, i, nm_rxq) {  in cxgbe_netmap_off()
837 		nm_state = atomic_load_int(&nm_rxq->nm_state);  in cxgbe_netmap_off()
838 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_off()
844 		MPASS(nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_off()
847 		    nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, 0xffff);  in cxgbe_netmap_off()
854 		while (!atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_OFF))  in cxgbe_netmap_off()
1193 	struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[vi->first_nm_rxq + kring->ring_id];  in cxgbe_netmap_rxsync()  local
1202 		kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);  in cxgbe_netmap_rxsync()
1206 	if (nm_rxq->fl_db_saved > 0 && starve_fl == 0) {  in cxgbe_netmap_rxsync()
1209 		    nm_rxq->fl_db_val | V_PIDX(nm_rxq->fl_db_saved));  in cxgbe_netmap_rxsync()
1210 		nm_rxq->fl_db_saved = 0;  in cxgbe_netmap_rxsync()
1218 		u_int fl_pidx = nm_rxq->fl_pidx;  in cxgbe_netmap_rxsync()
1221 		int i, dbinc = 0, hwidx = nm_rxq->fl_hwidx;  in cxgbe_netmap_rxsync()
1232 		IDXINCR(nm_rxq->fl_pidx, n, nm_rxq->fl_sidx2);  in cxgbe_netmap_rxsync()
1238 				nm_rxq->fl_desc[fl_pidx] = htobe64(ba | hwidx);  in cxgbe_netmap_rxsync()
1240 				MPASS(fl_pidx <= nm_rxq->fl_sidx2);  in cxgbe_netmap_rxsync()
1243 			if (fl_pidx == nm_rxq->fl_sidx2) {  in cxgbe_netmap_rxsync()
1247 			if (++dbinc == nm_rxq->fl_db_threshold) {  in cxgbe_netmap_rxsync()
1250 					nm_rxq->fl_db_saved += dbinc;  in cxgbe_netmap_rxsync()
1253 					    nm_rxq->fl_db_val | V_PIDX(dbinc));  in cxgbe_netmap_rxsync()
1258 		MPASS(nm_rxq->fl_pidx == fl_pidx);  in cxgbe_netmap_rxsync()
1263 				nm_rxq->fl_db_saved += dbinc;  in cxgbe_netmap_rxsync()
1266 				    nm_rxq->fl_db_val | V_PIDX(dbinc));  in cxgbe_netmap_rxsync()
1346 service_nm_rxq(struct sge_nm_rxq *nm_rxq)  in service_nm_rxq()  argument
1348 	struct vi_info *vi = nm_rxq->vi;  in service_nm_rxq()
1352 	struct netmap_kring *kring = na->rx_rings[nm_rxq->nid];  in service_nm_rxq()
1354 	struct iq_desc *d = &nm_rxq->iq_desc[nm_rxq->iq_cidx];  in service_nm_rxq()
1359 	uint32_t fl_cidx = atomic_load_acq_32(&nm_rxq->fl_cidx);  in service_nm_rxq()
1364 	while ((d->rsp.u.type_gen & F_RSPD_GEN) == nm_rxq->iq_gen) {  in service_nm_rxq()
1398 				if (__predict_false(++fl_cidx == nm_rxq->fl_sidx))  in service_nm_rxq()
1403 				    __func__, opcode, nm_rxq);  in service_nm_rxq()
1414 			    __func__, G_RSPD_TYPE(d->rsp.u.type_gen), nm_rxq);  in service_nm_rxq()
1418 		if (__predict_false(++nm_rxq->iq_cidx == nm_rxq->iq_sidx)) {  in service_nm_rxq()
1419 			nm_rxq->iq_cidx = 0;  in service_nm_rxq()
1420 			d = &nm_rxq->iq_desc[0];  in service_nm_rxq()
1421 			nm_rxq->iq_gen ^= F_RSPD_GEN;  in service_nm_rxq()
1425 			atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);  in service_nm_rxq()
1426 			netmap_rx_irq(ifp, nm_rxq->nid, &work);  in service_nm_rxq()
1433 				IDXINCR(nm_rxq->fl_pidx, fl_credits * 8,  in service_nm_rxq()
1434 				    nm_rxq->fl_sidx);  in service_nm_rxq()
1436 				    nm_rxq->fl_db_val | V_PIDX(fl_credits));  in service_nm_rxq()
1441 			    V_INGRESSQID(nm_rxq->iq_cntxt_id) |  in service_nm_rxq()
1447 	atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);  in service_nm_rxq()
1450 		IDXINCR(nm_rxq->fl_pidx, fl_credits * 8, nm_rxq->fl_sidx);  in service_nm_rxq()
1452 		    nm_rxq->fl_db_val | V_PIDX(fl_credits));  in service_nm_rxq()
1454 		netmap_rx_irq(ifp, nm_rxq->nid, &work);  in service_nm_rxq()
1457 	    V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) |  in service_nm_rxq()