42 	struct idxd_wq *wq;  member
54 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid);
100 	struct idxd_wq *wq = ctx->wq;  in cdev_file_attr_visible()  local
102 	if (!wq_pasid_enabled(wq))  in cdev_file_attr_visible()
121 	struct idxd_wq *wq = ctx->wq;  in idxd_file_dev_release()  local
122 	struct idxd_device *idxd = wq->idxd;  in idxd_file_dev_release()
128 	if (wq_shared(wq)) {  in idxd_file_dev_release()
133 			rc = idxd_wq_disable_pasid(wq);  in idxd_file_dev_release()
137 			idxd_wq_drain(wq);  in idxd_file_dev_release()
142 		idxd_cdev_evl_drain_pasid(wq, ctx->pasid);  in idxd_file_dev_release()
147 	mutex_lock(&wq->wq_lock);  in idxd_file_dev_release()
148 	idxd_wq_put(wq);  in idxd_file_dev_release()
149 	mutex_unlock(&wq->wq_lock);  in idxd_file_dev_release()
162 	struct idxd_wq *wq = idxd_cdev->wq;  in idxd_cdev_dev_release()  local
164 	cdev_ctx = &ictx[wq->idxd->data->type];  in idxd_cdev_dev_release()
185 	return idxd_cdev->wq;  in inode_wq()
190 	struct idxd_wq *wq = ctx->wq;  in idxd_xa_pasid_remove()  local
193 	mutex_lock(&wq->uc_lock);  in idxd_xa_pasid_remove()
194 	ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL);  in idxd_xa_pasid_remove()
196 		dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n",  in idxd_xa_pasid_remove()
198 	mutex_unlock(&wq->uc_lock);  in idxd_xa_pasid_remove()
201 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index)  in idxd_user_counter_increment()  argument
208 	mutex_lock(&wq->uc_lock);  in idxd_user_counter_increment()
209 	ctx = xa_load(&wq->upasid_xa, pasid);  in idxd_user_counter_increment()
211 		mutex_unlock(&wq->uc_lock);  in idxd_user_counter_increment()
215 	mutex_unlock(&wq->uc_lock);  in idxd_user_counter_increment()
222 	struct idxd_wq *wq;  in idxd_cdev_open()  local
229 	wq = inode_wq(inode);  in idxd_cdev_open()
230 	idxd = wq->idxd;  in idxd_cdev_open()
233 	dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));  in idxd_cdev_open()
239 	mutex_lock(&wq->wq_lock);  in idxd_cdev_open()
241 	if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {  in idxd_cdev_open()
246 	ctx->wq = wq;  in idxd_cdev_open()
268 		mutex_lock(&wq->uc_lock);  in idxd_cdev_open()
269 		rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL);  in idxd_cdev_open()
270 		mutex_unlock(&wq->uc_lock);  in idxd_cdev_open()
274 		if (wq_dedicated(wq)) {  in idxd_cdev_open()
275 			rc = idxd_wq_set_pasid(wq, pasid);  in idxd_cdev_open()
283 	idxd_cdev = wq->idxd_cdev;  in idxd_cdev_open()
308 	idxd_wq_get(wq);  in idxd_cdev_open()
309 	mutex_unlock(&wq->wq_lock);  in idxd_cdev_open()
323 	mutex_unlock(&wq->wq_lock);  in idxd_cdev_open()
328 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid)  in idxd_cdev_evl_drain_pasid()  argument
330 	struct idxd_device *idxd = wq->idxd;  in idxd_cdev_evl_drain_pasid()
348 		if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id)  in idxd_cdev_evl_drain_pasid()
352 	drain_workqueue(wq->wq);  in idxd_cdev_evl_drain_pasid()
359 	struct idxd_wq *wq = ctx->wq;  in idxd_cdev_release()  local
360 	struct idxd_device *idxd = wq->idxd;  in idxd_cdev_release()
371 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,  in check_vma()  argument
374 	struct device *dev = &wq->idxd->pdev->dev;  in check_vma()
390 	struct idxd_wq *wq = ctx->wq;  in idxd_cdev_mmap()  local
391 	struct idxd_device *idxd = wq->idxd;  in idxd_cdev_mmap()
413 	rc = check_vma(wq, vma, __func__);  in idxd_cdev_mmap()
418 	pfn = (base + idxd_get_wq_portal_full_offset(wq->id,  in idxd_cdev_mmap()
430 	struct idxd_wq *wq = ctx->wq;  in idxd_submit_user_descriptor()  local
431 	struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev;  in idxd_submit_user_descriptor()
433 	void __iomem *portal = idxd_wq_portal_addr(wq);  in idxd_submit_user_descriptor()
448 		!wq->idxd->user_submission_safe)  in idxd_submit_user_descriptor()
458 	if (wq_dedicated(wq))  in idxd_submit_user_descriptor()
463 		rc = idxd_enqcmds(wq, portal, &descriptor);  in idxd_submit_user_descriptor()
498 	struct idxd_wq *wq = ctx->wq;  in idxd_cdev_poll()  local
499 	struct idxd_device *idxd = wq->idxd;  in idxd_cdev_poll()
505 	poll_wait(filp, &wq->err_queue, wait);  in idxd_cdev_poll()
528 int idxd_wq_add_cdev(struct idxd_wq *wq)  in idxd_wq_add_cdev()  argument
530 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_add_cdev()
542 	idxd_cdev->wq = wq;  in idxd_wq_add_cdev()
545 	cdev_ctx = &ictx[wq->idxd->data->type];  in idxd_wq_add_cdev()
554 	dev->parent = wq_confdev(wq);  in idxd_wq_add_cdev()
559 	rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);  in idxd_wq_add_cdev()
563 	wq->idxd_cdev = idxd_cdev;  in idxd_wq_add_cdev()
567 		dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);  in idxd_wq_add_cdev()
575 	wq->idxd_cdev = NULL;  in idxd_wq_add_cdev()
579 void idxd_wq_del_cdev(struct idxd_wq *wq)  in idxd_wq_del_cdev()  argument
583 	idxd_cdev = wq->idxd_cdev;  in idxd_wq_del_cdev()
584 	wq->idxd_cdev = NULL;  in idxd_wq_del_cdev()
592 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);  in idxd_user_drv_probe()  local
593 	struct idxd_device *idxd = wq->idxd;  in idxd_user_drv_probe()
599 	mutex_lock(&wq->wq_lock);  in idxd_user_drv_probe()
601 	if (!idxd_wq_driver_name_match(wq, dev)) {  in idxd_user_drv_probe()
626 	wq->wq = create_workqueue(dev_name(wq_confdev(wq)));  in idxd_user_drv_probe()
627 	if (!wq->wq) {  in idxd_user_drv_probe()
632 	wq->type = IDXD_WQT_USER;  in idxd_user_drv_probe()
633 	rc = idxd_drv_enable_wq(wq);  in idxd_user_drv_probe()
637 	rc = idxd_wq_add_cdev(wq);  in idxd_user_drv_probe()
644 	mutex_unlock(&wq->wq_lock);  in idxd_user_drv_probe()
648 	idxd_drv_disable_wq(wq);  in idxd_user_drv_probe()
650 	destroy_workqueue(wq->wq);  in idxd_user_drv_probe()
651 	wq->type = IDXD_WQT_NONE;  in idxd_user_drv_probe()
653 	mutex_unlock(&wq->wq_lock);  in idxd_user_drv_probe()
659 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);  in idxd_user_drv_remove()  local
661 	mutex_lock(&wq->wq_lock);  in idxd_user_drv_remove()
662 	idxd_wq_del_cdev(wq);  in idxd_user_drv_remove()
663 	idxd_drv_disable_wq(wq);  in idxd_user_drv_remove()
664 	wq->type = IDXD_WQT_NONE;  in idxd_user_drv_remove()
665 	destroy_workqueue(wq->wq);  in idxd_user_drv_remove()
666 	wq->wq = NULL;  in idxd_user_drv_remove()
667 	mutex_unlock(&wq->wq_lock);  in idxd_user_drv_remove()
727 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,  in idxd_copy_cr()  argument
730 	struct device *dev = &wq->idxd->pdev->dev;  in idxd_copy_cr()
735 	mutex_lock(&wq->uc_lock);  in idxd_copy_cr()
737 	ctx = xa_load(&wq->upasid_xa, pasid);  in idxd_copy_cr()
776 	mutex_unlock(&wq->uc_lock);  in idxd_copy_cr()