54 pf_af_sync_msg(struct dev *dev, struct mbox_msghdr **rsp)  in pf_af_sync_msg()  argument
57 	struct mbox *mbox = dev->mbox;  in pf_af_sync_msg()
58 	struct mbox_dev *mdev = &mbox->dev[0];  in pf_af_sync_msg()
66 	plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C);  in pf_af_sync_msg()
79 		int_status = plt_read64(dev->bar2 + RVU_PF_INT);  in pf_af_sync_msg()
83 	plt_write64(int_status, dev->bar2 + RVU_PF_INT);  in pf_af_sync_msg()
86 	plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1S);  in pf_af_sync_msg()
102 af_pf_wait_msg(struct dev *dev, uint16_t vf, int num_msg)  in af_pf_wait_msg()  argument
105 	struct mbox *mbox = dev->mbox;  in af_pf_wait_msg()
106 	struct mbox_dev *mdev = &mbox->dev[0];  in af_pf_wait_msg()
116 	plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C);  in af_pf_wait_msg()
129 		int_status = plt_read64(dev->bar2 + RVU_PF_INT);  in af_pf_wait_msg()
133 	plt_write64(~0ull, dev->bar2 + RVU_PF_INT);  in af_pf_wait_msg()
136 	plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1S);  in af_pf_wait_msg()
154 		rsp = mbox_alloc_msg(&dev->mbox_vfpf, vf, size);  in af_pf_wait_msg()
184 			if (dev->ops && dev->ops->link_status_get)  in af_pf_wait_msg()
185 				dev->ops->link_status_get(dev->roc_nix, &linfo);  in af_pf_wait_msg()
190 			vf_msg = mbox_alloc_msg(&dev->mbox_vfpf_up, vf, sz);  in af_pf_wait_msg()
200 				mbox_msg_send(&dev->mbox_vfpf_up, vf);  in af_pf_wait_msg()
212 vf_pf_process_msgs(struct dev *dev, uint16_t vf)  in vf_pf_process_msgs()  argument
214 	struct mbox *mbox = &dev->mbox_vfpf;  in vf_pf_process_msgs()
215 	struct mbox_dev *mdev = &mbox->dev[vf];  in vf_pf_process_msgs()
233 		msg->pcifunc = dev_pf_func(dev->pf, vf);  in vf_pf_process_msgs()
237 			uint16_t max_bits = sizeof(dev->active_vfs[0]) * 8;  in vf_pf_process_msgs()
240 			dev->active_vfs[vf / max_bits] |=  in vf_pf_process_msgs()
259 			af_req = mbox_alloc_msg(dev->mbox, 0, size);  in vf_pf_process_msgs()
277 			     dev->pf, routed, vf);  in vf_pf_process_msgs()
278 		af_pf_wait_msg(dev, vf, routed);  in vf_pf_process_msgs()
279 		mbox_reset(dev->mbox, 0);  in vf_pf_process_msgs()
284 		plt_base_dbg("pf:%d reply %d messages to vf:%d", dev->pf,  in vf_pf_process_msgs()
293 vf_pf_process_up_msgs(struct dev *dev, uint16_t vf)  in vf_pf_process_up_msgs()  argument
295 	struct mbox *mbox = &dev->mbox_vfpf_up;  in vf_pf_process_up_msgs()
296 	struct mbox_dev *mdev = &mbox->dev[vf];  in vf_pf_process_up_msgs()
314 		msg->pcifunc = dev_pf_func(dev->pf, vf);  in vf_pf_process_up_msgs()
346 	struct dev *dev = param;  in roc_vf_pf_mbox_handle_msg()  local
348 	max_bits = sizeof(dev->intr.bits[0]) * sizeof(uint64_t);  in roc_vf_pf_mbox_handle_msg()
352 		if (dev->intr.bits[vf / max_bits] & BIT_ULL(vf % max_bits)) {  in roc_vf_pf_mbox_handle_msg()
354 				     dev->pf, dev->vf);  in roc_vf_pf_mbox_handle_msg()
355 			vf_pf_process_msgs(dev, vf);  in roc_vf_pf_mbox_handle_msg()
357 			vf_pf_process_up_msgs(dev, vf);  in roc_vf_pf_mbox_handle_msg()
358 			dev->intr.bits[vf / max_bits] &=  in roc_vf_pf_mbox_handle_msg()
362 	dev->timer_set = 0;  in roc_vf_pf_mbox_handle_msg()
368 	struct dev *dev = param;  in roc_vf_pf_mbox_irq()  local
374 		intr = plt_read64(dev->bar2 + RVU_PF_VFPF_MBOX_INTX(vfpf));  in roc_vf_pf_mbox_irq()
379 			     vfpf, intr, dev->pf, dev->vf);  in roc_vf_pf_mbox_irq()
382 		dev->intr.bits[vfpf] |= intr;  in roc_vf_pf_mbox_irq()
383 		plt_write64(intr, dev->bar2 + RVU_PF_VFPF_MBOX_INTX(vfpf));  in roc_vf_pf_mbox_irq()
387 	if (!dev->timer_set && alarm_set) {  in roc_vf_pf_mbox_irq()
388 		dev->timer_set = 1;  in roc_vf_pf_mbox_irq()
391 			      dev);  in roc_vf_pf_mbox_irq()
396 process_msgs(struct dev *dev, struct mbox *mbox)  in process_msgs()  argument
398 	struct mbox_dev *mdev = &mbox->dev[0];  in process_msgs()
422 			dev->pf_func = msg->pcifunc;  in process_msgs()
442 pf_vf_mbox_send_up_msg(struct dev *dev, void *rec_msg)  in pf_vf_mbox_send_up_msg()  argument
444 	uint16_t max_bits = sizeof(dev->active_vfs[0]) * sizeof(uint64_t);  in pf_vf_mbox_send_up_msg()
445 	struct mbox *vf_mbox = &dev->mbox_vfpf_up;  in pf_vf_mbox_send_up_msg()
455 		if (!(dev->active_vfs[vf / max_bits] & (BIT_ULL(vf))))  in pf_vf_mbox_send_up_msg()
479 		vf_msg->pcifunc = dev->pf_func;  in pf_vf_mbox_send_up_msg()
487 mbox_up_handler_cgx_link_event(struct dev *dev, struct cgx_link_info_msg *msg,  in mbox_up_handler_cgx_link_event()  argument
491 	void *roc_nix = dev->roc_nix;  in mbox_up_handler_cgx_link_event()
494 		     dev_get_pf(dev->pf_func), dev_get_vf(dev->pf_func),  in mbox_up_handler_cgx_link_event()
501 		if (dev->ops && dev->ops->link_status_update)  in mbox_up_handler_cgx_link_event()
502 			dev->ops->link_status_update(roc_nix, linfo);  in mbox_up_handler_cgx_link_event()
505 		pf_vf_mbox_send_up_msg(dev, msg);  in mbox_up_handler_cgx_link_event()
508 		if (dev->ops && dev->ops->link_status_update)  in mbox_up_handler_cgx_link_event()
509 			dev->ops->link_status_update(roc_nix, linfo);  in mbox_up_handler_cgx_link_event()
517 mbox_up_handler_cgx_ptp_rx_info(struct dev *dev,  in mbox_up_handler_cgx_ptp_rx_info()  argument
521 	void *roc_nix = dev->roc_nix;  in mbox_up_handler_cgx_ptp_rx_info()
524 		     dev_get_pf(dev->pf_func), dev_get_vf(dev->pf_func),  in mbox_up_handler_cgx_ptp_rx_info()
531 		if (dev->ops && dev->ops->ptp_info_update)  in mbox_up_handler_cgx_ptp_rx_info()
532 			dev->ops->ptp_info_update(roc_nix, msg->ptp_en);  in mbox_up_handler_cgx_ptp_rx_info()
535 		pf_vf_mbox_send_up_msg(dev, msg);  in mbox_up_handler_cgx_ptp_rx_info()
538 		if (dev->ops && dev->ops->ptp_info_update)  in mbox_up_handler_cgx_ptp_rx_info()
539 			dev->ops->ptp_info_update(roc_nix, msg->ptp_en);  in mbox_up_handler_cgx_ptp_rx_info()
547 mbox_process_msgs_up(struct dev *dev, struct mbox_msghdr *req)  in mbox_process_msgs_up()  argument
555 		reply_invalid_msg(&dev->mbox_up, 0, 0, req->id);  in mbox_process_msgs_up()
562 			&dev->mbox_up, 0, sizeof(struct _rsp_type));           \  in mbox_process_msgs_up()
567 		rsp->hdr.pcifunc = dev->pf_func;                               \  in mbox_process_msgs_up()
569 		err = mbox_up_handler_##_fn_name(dev, (struct _req_type *)req, \  in mbox_process_msgs_up()
581 process_msgs_up(struct dev *dev, struct mbox *mbox)  in process_msgs_up()  argument
583 	struct mbox_dev *mdev = &mbox->dev[0];  in process_msgs_up()
599 		err = mbox_process_msgs_up(dev, msg);  in process_msgs_up()
615 	struct dev *dev = param;  in roc_pf_vf_mbox_irq()  local
618 	intr = plt_read64(dev->bar2 + RVU_VF_INT);  in roc_pf_vf_mbox_irq()
622 	plt_write64(intr, dev->bar2 + RVU_VF_INT);  in roc_pf_vf_mbox_irq()
623 	plt_base_dbg("Irq 0x%" PRIx64 "(pf:%d,vf:%d)", intr, dev->pf, dev->vf);  in roc_pf_vf_mbox_irq()
626 	process_msgs(dev, dev->mbox);  in roc_pf_vf_mbox_irq()
629 	process_msgs_up(dev, &dev->mbox_up);  in roc_pf_vf_mbox_irq()
635 	struct dev *dev = param;  in roc_af_pf_mbox_irq()  local
638 	intr = plt_read64(dev->bar2 + RVU_PF_INT);  in roc_af_pf_mbox_irq()
642 	plt_write64(intr, dev->bar2 + RVU_PF_INT);  in roc_af_pf_mbox_irq()
643 	plt_base_dbg("Irq 0x%" PRIx64 "(pf:%d,vf:%d)", intr, dev->pf, dev->vf);  in roc_af_pf_mbox_irq()
646 	process_msgs(dev, dev->mbox);  in roc_af_pf_mbox_irq()
649 	process_msgs_up(dev, &dev->mbox_up);  in roc_af_pf_mbox_irq()
653 mbox_register_pf_irq(struct plt_pci_device *pci_dev, struct dev *dev)  in mbox_register_pf_irq()  argument
661 			    dev->bar2 + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(i));  in mbox_register_pf_irq()
663 	plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C);  in mbox_register_pf_irq()
665 	dev->timer_set = 0;  in mbox_register_pf_irq()
668 	rc = dev_irq_register(intr_handle, roc_vf_pf_mbox_irq, dev,  in mbox_register_pf_irq()
676 	rc = dev_irq_register(intr_handle, roc_vf_pf_mbox_irq, dev,  in mbox_register_pf_irq()
684 	rc = dev_irq_register(intr_handle, roc_af_pf_mbox_irq, dev,  in mbox_register_pf_irq()
694 			    dev->bar2 + RVU_PF_VFPF_MBOX_INT_ENA_W1SX(i));  in mbox_register_pf_irq()
696 	plt_write64(~0ull, dev->bar2 + RVU_PF_INT);  in mbox_register_pf_irq()
697 	plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1S);  in mbox_register_pf_irq()
703 mbox_register_vf_irq(struct plt_pci_device *pci_dev, struct dev *dev)  in mbox_register_vf_irq()  argument
709 	plt_write64(~0ull, dev->bar2 + RVU_VF_INT_ENA_W1C);  in mbox_register_vf_irq()
712 	rc = dev_irq_register(intr_handle, roc_pf_vf_mbox_irq, dev,  in mbox_register_vf_irq()
720 	plt_write64(~0ull, dev->bar2 + RVU_VF_INT);  in mbox_register_vf_irq()
721 	plt_write64(~0ull, dev->bar2 + RVU_VF_INT_ENA_W1S);  in mbox_register_vf_irq()
727 mbox_register_irq(struct plt_pci_device *pci_dev, struct dev *dev)  in mbox_register_irq()  argument
729 	if (dev_is_vf(dev))  in mbox_register_irq()
730 		return mbox_register_vf_irq(pci_dev, dev);  in mbox_register_irq()
732 		return mbox_register_pf_irq(pci_dev, dev);  in mbox_register_irq()
736 mbox_unregister_pf_irq(struct plt_pci_device *pci_dev, struct dev *dev)  in mbox_unregister_pf_irq()  argument
744 			    dev->bar2 + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(i));  in mbox_unregister_pf_irq()
746 	plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C);  in mbox_unregister_pf_irq()
748 	dev->timer_set = 0;  in mbox_unregister_pf_irq()
750 	plt_alarm_cancel(roc_vf_pf_mbox_handle_msg, dev);  in mbox_unregister_pf_irq()
754 	dev_irq_unregister(intr_handle, roc_vf_pf_mbox_irq, dev,  in mbox_unregister_pf_irq()
758 	dev_irq_unregister(intr_handle, roc_vf_pf_mbox_irq, dev,  in mbox_unregister_pf_irq()
762 	dev_irq_unregister(intr_handle, roc_af_pf_mbox_irq, dev,  in mbox_unregister_pf_irq()
767 mbox_unregister_vf_irq(struct plt_pci_device *pci_dev, struct dev *dev)  in mbox_unregister_vf_irq()  argument
772 	plt_write64(~0ull, dev->bar2 + RVU_VF_INT_ENA_W1C);  in mbox_unregister_vf_irq()
775 	dev_irq_unregister(intr_handle, roc_pf_vf_mbox_irq, dev,  in mbox_unregister_vf_irq()
780 mbox_unregister_irq(struct plt_pci_device *pci_dev, struct dev *dev)  in mbox_unregister_irq()  argument
782 	if (dev_is_vf(dev))  in mbox_unregister_irq()
783 		mbox_unregister_vf_irq(pci_dev, dev);  in mbox_unregister_irq()
785 		mbox_unregister_pf_irq(pci_dev, dev);  in mbox_unregister_irq()
789 vf_flr_send_msg(struct dev *dev, uint16_t vf)  in vf_flr_send_msg()  argument
791 	struct mbox *mbox = dev->mbox;  in vf_flr_send_msg()
799 	req->hdr.pcifunc = dev_pf_func(dev->pf, vf);  in vf_flr_send_msg()
802 	rc = pf_af_sync_msg(dev, NULL);  in vf_flr_send_msg()
812 	struct dev *dev = (struct dev *)param;  in roc_pf_vf_flr_irq()  local
818 	max_vf = (dev->maxvf > 0) ? dev->maxvf : 64;  in roc_pf_vf_flr_irq()
819 	bar2 = dev->bar2;  in roc_pf_vf_flr_irq()
840 			vf_flr_send_msg(dev, vf);  in roc_pf_vf_flr_irq()
851 vf_flr_unregister_irqs(struct plt_pci_device *pci_dev, struct dev *dev)  in vf_flr_unregister_irqs()  argument
860 		plt_write64(~0ull, dev->bar2 + RVU_PF_VFFLR_INT_ENA_W1CX(i));  in vf_flr_unregister_irqs()
862 	dev_irq_unregister(intr_handle, roc_pf_vf_flr_irq, dev,  in vf_flr_unregister_irqs()
865 	dev_irq_unregister(intr_handle, roc_pf_vf_flr_irq, dev,  in vf_flr_unregister_irqs()
872 vf_flr_register_irqs(struct plt_pci_device *pci_dev, struct dev *dev)  in vf_flr_register_irqs()  argument
879 	rc = dev_irq_register(handle, roc_pf_vf_flr_irq, dev,  in vf_flr_register_irqs()
884 	rc = dev_irq_register(handle, roc_pf_vf_flr_irq, dev,  in vf_flr_register_irqs()
891 		plt_write64(~0ull, dev->bar2 + RVU_PF_VFFLR_INTX(i));  in vf_flr_register_irqs()
892 		plt_write64(~0ull, dev->bar2 + RVU_PF_VFTRPENDX(i));  in vf_flr_register_irqs()
893 		plt_write64(~0ull, dev->bar2 + RVU_PF_VFFLR_INT_ENA_W1SX(i));  in vf_flr_register_irqs()
899 clear_rvum_interrupts(struct dev *dev)  in clear_rvum_interrupts()  argument
904 	if (dev_is_vf(dev)) {  in clear_rvum_interrupts()
906 		intr = plt_read64(dev->bar2 + RVU_VF_INT);  in clear_rvum_interrupts()
908 			plt_write64(intr, dev->bar2 + RVU_VF_INT);  in clear_rvum_interrupts()
911 		intr = plt_read64(dev->bar2 + RVU_PF_INT);  in clear_rvum_interrupts()
913 			plt_write64(intr, dev->bar2 + RVU_PF_INT);  in clear_rvum_interrupts()
916 			intr = plt_read64(dev->bar2 + RVU_PF_VFPF_MBOX_INTX(i));  in clear_rvum_interrupts()
919 					    dev->bar2 +  in clear_rvum_interrupts()
922 			intr = plt_read64(dev->bar2 + RVU_PF_VFFLR_INTX(i));  in clear_rvum_interrupts()
925 					    dev->bar2 + RVU_PF_VFFLR_INTX(i));  in clear_rvum_interrupts()
931 dev_active_vfs(struct dev *dev)  in dev_active_vfs()  argument
936 		count += __builtin_popcount(dev->active_vfs[i]);  in dev_active_vfs()
942 dev_vf_hwcap_update(struct plt_pci_device *pci_dev, struct dev *dev)  in dev_vf_hwcap_update()  argument
954 		dev->hwcap |= DEV_HWCAP_F_VF;  in dev_vf_hwcap_update()
960 dev_vf_mbase_get(struct plt_pci_device *pci_dev, struct dev *dev)  in dev_vf_mbase_get()  argument
965 	if (dev_is_vf(dev))  in dev_vf_mbase_get()
970 		return dev->bar4 + MBOX_SIZE;  in dev_vf_mbase_get()
972 	pa = plt_read64(dev->bar2 + RVU_PF_VF_BAR4_ADDR);  in dev_vf_mbase_get()
1019 dev_lmt_setup(struct dev *dev)  in dev_lmt_setup()  argument
1027 		dev->lmt_base = dev->bar2 + (RVU_BLOCK_ADDR_LMT << 20);  in dev_lmt_setup()
1034 	if (!dev->disable_shared_lmt && idev_lmt_pffunc_get() &&  in dev_lmt_setup()
1035 	    dev->pf_func != idev_lmt_pffunc_get()) {  in dev_lmt_setup()
1036 		rc = dev_setup_shared_lmt_region(dev->mbox, false, 0);  in dev_lmt_setup()
1041 			dev->lmt_base = roc_idev_lmt_base_addr_get();  in dev_lmt_setup()
1046 			dev->pf_func, rc);  in dev_lmt_setup()
1050 	sprintf(name, "LMT_MAP%x", dev->pf_func);  in dev_lmt_setup()
1065 	rc = dev_setup_shared_lmt_region(dev->mbox, true, mz->iova);  in dev_lmt_setup()
1071 	dev->lmt_base = mz->iova;  in dev_lmt_setup()
1072 	dev->lmt_mz = mz;  in dev_lmt_setup()
1076 	if (!dev->disable_shared_lmt) {  in dev_lmt_setup()
1084 			idev->lmt_base_addr = dev->lmt_base;  in dev_lmt_setup()
1085 			idev->lmt_pf_func = dev->pf_func;  in dev_lmt_setup()
1098 dev_init(struct dev *dev, struct plt_pci_device *pci_dev)  in dev_init()  argument
1126 	dev->maxvf = pci_dev->max_vfs;  in dev_init()
1127 	dev->bar2 = bar2;  in dev_init()
1128 	dev->bar4 = bar4;  in dev_init()
1129 	dev_vf_hwcap_update(pci_dev, dev);  in dev_init()
1131 	if (dev_is_vf(dev)) {  in dev_init()
1145 	clear_rvum_interrupts(dev);  in dev_init()
1148 	rc = mbox_init(&dev->mbox_local, mbox, bar2, direction, 1, intr_offset);  in dev_init()
1151 	dev->mbox = &dev->mbox_local;  in dev_init()
1153 	rc = mbox_init(&dev->mbox_up, mbox, bar2, up_direction, 1, intr_offset);  in dev_init()
1158 	rc = mbox_register_irq(pci_dev, dev);  in dev_init()
1163 	rc = send_ready_msg(dev->mbox, &dev->pf_func);  in dev_init()
1167 	dev->pf = dev_get_pf(dev->pf_func);  in dev_init()
1168 	dev->vf = dev_get_vf(dev->pf_func);  in dev_init()
1169 	memset(&dev->active_vfs, 0, sizeof(dev->active_vfs));  in dev_init()
1172 	dev->ops = plt_zmalloc(sizeof(struct dev_ops), 0);  in dev_init()
1173 	if (dev->ops == NULL) {  in dev_init()
1181 		vf_mbase = dev_vf_mbase_get(pci_dev, dev);  in dev_init()
1187 		rc = mbox_init(&dev->mbox_vfpf, vf_mbase, bar2, MBOX_DIR_PFVF,  in dev_init()
1193 		rc = mbox_init(&dev->mbox_vfpf_up, vf_mbase, bar2,  in dev_init()
1200 	if (!dev_is_vf(dev)) {  in dev_init()
1201 		rc = vf_flr_register_irqs(pci_dev, dev);  in dev_init()
1205 	dev->mbox_active = 1;  in dev_init()
1207 	rc = npa_lf_init(dev, pci_dev);  in dev_init()
1212 	rc = dev_lmt_setup(dev);  in dev_init()
1220 	mbox_unregister_irq(pci_dev, dev);  in dev_init()
1221 	if (dev->ops)  in dev_init()
1222 		plt_free(dev->ops);  in dev_init()
1224 	mbox_fini(dev->mbox);  in dev_init()
1225 	mbox_fini(&dev->mbox_up);  in dev_init()
1231 dev_fini(struct dev *dev, struct plt_pci_device *pci_dev)  in dev_fini()  argument
1237 	if (idev_npa_lf_active(dev) > 1)  in dev_fini()
1244 	if (dev->lmt_mz)  in dev_fini()
1245 		plt_memzone_free(dev->lmt_mz);  in dev_fini()
1247 	mbox_unregister_irq(pci_dev, dev);  in dev_fini()
1249 	if (!dev_is_vf(dev))  in dev_fini()
1250 		vf_flr_unregister_irqs(pci_dev, dev);  in dev_fini()
1252 	mbox = &dev->mbox_vfpf;  in dev_fini()
1253 	if (mbox->hwbase && mbox->dev)  in dev_fini()
1256 	if (dev->ops)  in dev_fini()
1257 		plt_free(dev->ops);  in dev_fini()
1260 	mbox = &dev->mbox_vfpf_up;  in dev_fini()
1264 	mbox = dev->mbox;  in dev_fini()
1266 	mbox = &dev->mbox_up;  in dev_fini()
1268 	dev->mbox_active = 0;  in dev_fini()