446 	struct mlx5_common_device *cdev;  in to_mlx5_device()  local
448 	TAILQ_FOREACH(cdev, &devices_list, next) {  in to_mlx5_device()
449 		if (rte_dev == cdev->dev)  in to_mlx5_device()
450 			return cdev;  in to_mlx5_device()
490 mlx5_dev_mempool_register(struct mlx5_common_device *cdev,  in mlx5_dev_mempool_register()  argument
493 	return mlx5_mr_mempool_register(cdev, mp, is_extmem);  in mlx5_dev_mempool_register()
505 mlx5_dev_mempool_unregister(struct mlx5_common_device *cdev,  in mlx5_dev_mempool_unregister()  argument
508 	if (mlx5_mr_mempool_unregister(cdev, mp) < 0)  in mlx5_dev_mempool_unregister()
510 			mp->name, cdev->pd, rte_strerror(rte_errno));  in mlx5_dev_mempool_unregister()
524 	struct mlx5_common_device *cdev = arg;  in mlx5_dev_mempool_register_cb()  local
527 	ret = mlx5_dev_mempool_register(cdev, mp, false);  in mlx5_dev_mempool_register_cb()
531 			mp->name, cdev->pd, rte_strerror(rte_errno));  in mlx5_dev_mempool_register_cb()
563 	struct mlx5_common_device *cdev = arg;  in mlx5_dev_mempool_event_cb()  local
567 		if (mlx5_dev_mempool_register(cdev, mp, false) < 0)  in mlx5_dev_mempool_event_cb()
570 				mp->name, cdev->pd, rte_strerror(rte_errno));  in mlx5_dev_mempool_event_cb()
573 		mlx5_dev_mempool_unregister(cdev, mp);  in mlx5_dev_mempool_event_cb()
579 mlx5_dev_mempool_subscribe(struct mlx5_common_device *cdev)  in mlx5_dev_mempool_subscribe()  argument
583 	if (!cdev->config.mr_mempool_reg_en)  in mlx5_dev_mempool_subscribe()
585 	rte_rwlock_write_lock(&cdev->mr_scache.mprwlock);  in mlx5_dev_mempool_subscribe()
586 	if (cdev->mr_scache.mp_cb_registered)  in mlx5_dev_mempool_subscribe()
590 						  cdev);  in mlx5_dev_mempool_subscribe()
595 		rte_mempool_walk(mlx5_dev_mempool_register_cb, cdev);  in mlx5_dev_mempool_subscribe()
597 	cdev->mr_scache.mp_cb_registered = 1;  in mlx5_dev_mempool_subscribe()
599 	rte_rwlock_write_unlock(&cdev->mr_scache.mprwlock);  in mlx5_dev_mempool_subscribe()
604 mlx5_dev_mempool_unsubscribe(struct mlx5_common_device *cdev)  in mlx5_dev_mempool_unsubscribe()  argument
608 	if (!cdev->mr_scache.mp_cb_registered ||  in mlx5_dev_mempool_unsubscribe()
609 	    !cdev->config.mr_mempool_reg_en)  in mlx5_dev_mempool_unsubscribe()
613 						    cdev);  in mlx5_dev_mempool_unsubscribe()
615 		rte_mempool_walk(mlx5_dev_mempool_unregister_cb, cdev);  in mlx5_dev_mempool_unsubscribe()
632 	struct mlx5_common_device *cdev;  in mlx5_mr_mem_event_cb()  local
640 		TAILQ_FOREACH(cdev, &devices_list, next)  in mlx5_mr_mem_event_cb()
641 			mlx5_free_mr_by_addr(&cdev->mr_scache,  in mlx5_mr_mem_event_cb()
643 								    (cdev->ctx),  in mlx5_mr_mem_event_cb()
663 mlx5_dev_hw_global_release(struct mlx5_common_device *cdev)  in mlx5_dev_hw_global_release()  argument
665 	if (cdev->pd != NULL) {  in mlx5_dev_hw_global_release()
666 		claim_zero(mlx5_os_pd_release(cdev));  in mlx5_dev_hw_global_release()
667 		cdev->pd = NULL;  in mlx5_dev_hw_global_release()
669 	if (cdev->ctx != NULL) {  in mlx5_dev_hw_global_release()
670 		claim_zero(mlx5_glue->close_device(cdev->ctx));  in mlx5_dev_hw_global_release()
671 		cdev->ctx = NULL;  in mlx5_dev_hw_global_release()
687 mlx5_dev_hw_global_prepare(struct mlx5_common_device *cdev, uint32_t classes)  in mlx5_dev_hw_global_prepare()  argument
692 	ret = mlx5_os_open_device(cdev, classes);  in mlx5_dev_hw_global_prepare()
700 	if (cdev->config.devx || cdev->config.device_fd != MLX5_ARG_UNSET) {  in mlx5_dev_hw_global_prepare()
702 		ret = mlx5_devx_cmd_query_hca_attr(cdev->ctx,  in mlx5_dev_hw_global_prepare()
703 						   &cdev->config.hca_attr);  in mlx5_dev_hw_global_prepare()
709 		cdev->config.devx = 1;  in mlx5_dev_hw_global_prepare()
711 	DRV_LOG(DEBUG, "DevX is %ssupported.", cdev->config.devx ? "" : "NOT ");  in mlx5_dev_hw_global_prepare()
713 	ret = mlx5_os_pd_prepare(cdev);  in mlx5_dev_hw_global_prepare()
718 	mlx5_dev_hw_global_release(cdev);  in mlx5_dev_hw_global_prepare()
723 mlx5_common_dev_release(struct mlx5_common_device *cdev)  in mlx5_common_dev_release()  argument
726 	TAILQ_REMOVE(&devices_list, cdev, next);  in mlx5_common_dev_release()
732 		mlx5_dev_mempool_unsubscribe(cdev);  in mlx5_common_dev_release()
733 		mlx5_mr_release_cache(&cdev->mr_scache);  in mlx5_common_dev_release()
734 		mlx5_dev_hw_global_release(cdev);  in mlx5_common_dev_release()
736 	rte_free(cdev);  in mlx5_common_dev_release()
743 	struct mlx5_common_device *cdev;  in mlx5_common_dev_create()  local
746 	cdev = rte_zmalloc("mlx5_common_device", sizeof(*cdev), 0);  in mlx5_common_dev_create()
747 	if (!cdev) {  in mlx5_common_dev_create()
752 	cdev->dev = eal_dev;  in mlx5_common_dev_create()
756 	ret = mlx5_common_config_get(mkvlist, &cdev->config);  in mlx5_common_dev_create()
760 		rte_free(cdev);  in mlx5_common_dev_create()
763 	mlx5_malloc_mem_select(cdev->config.sys_mem_en);  in mlx5_common_dev_create()
765 	ret = mlx5_dev_hw_global_prepare(cdev, classes);  in mlx5_common_dev_create()
768 		rte_free(cdev);  in mlx5_common_dev_create()
772 	ret = mlx5_mr_create_cache(&cdev->mr_scache, eal_dev->numa_node);  in mlx5_common_dev_create()
775 		mlx5_dev_hw_global_release(cdev);  in mlx5_common_dev_create()
776 		rte_free(cdev);  in mlx5_common_dev_create()
785 	TAILQ_INSERT_HEAD(&devices_list, cdev, next);  in mlx5_common_dev_create()
787 	return cdev;  in mlx5_common_dev_create()
808 mlx5_common_probe_again_args_validate(struct mlx5_common_device *cdev,  in mlx5_common_probe_again_args_validate()  argument
842 	if (cdev->config.mr_ext_memseg_en != config->mr_ext_memseg_en) {  in mlx5_common_probe_again_args_validate()
845 			cdev->dev->name);  in mlx5_common_probe_again_args_validate()
848 	if (cdev->config.mr_mempool_reg_en != config->mr_mempool_reg_en) {  in mlx5_common_probe_again_args_validate()
851 			cdev->dev->name);  in mlx5_common_probe_again_args_validate()
854 	if (cdev->config.device_fd != config->device_fd) {  in mlx5_common_probe_again_args_validate()
857 			cdev->dev->name);  in mlx5_common_probe_again_args_validate()
860 	if (cdev->config.pd_handle != config->pd_handle) {  in mlx5_common_probe_again_args_validate()
863 			cdev->dev->name);  in mlx5_common_probe_again_args_validate()
866 	if (cdev->config.sys_mem_en != config->sys_mem_en) {  in mlx5_common_probe_again_args_validate()
869 			cdev->dev->name);  in mlx5_common_probe_again_args_validate()
872 	if (cdev->config.dbnc != config->dbnc) {  in mlx5_common_probe_again_args_validate()
875 			cdev->dev->name);  in mlx5_common_probe_again_args_validate()
887 drivers_remove(struct mlx5_common_device *cdev, uint32_t enabled_classes)  in drivers_remove()  argument
897 			local_ret = driver->remove(cdev);  in drivers_remove()
899 				cdev->classes_loaded &= ~RTE_BIT64(i);  in drivers_remove()
912 drivers_probe(struct mlx5_common_device *cdev, uint32_t user_classes,  in drivers_probe()  argument
923 		if (!mlx5_bus_match(driver, cdev->dev))  in drivers_probe()
925 		already_loaded = cdev->classes_loaded & driver->drv_class;  in drivers_probe()
928 				cdev->dev->name);  in drivers_probe()
932 		ret = driver->probe(cdev, mkvlist);  in drivers_probe()
941 		cdev->classes_loaded |= enabled_classes;  in drivers_probe()
949 	enabled_classes &= ~cdev->classes_loaded;  in drivers_probe()
950 	drivers_remove(cdev, enabled_classes);  in drivers_probe()
957 	struct mlx5_common_device *cdev;  in mlx5_common_dev_probe()  local
993 	cdev = to_mlx5_device(eal_dev);  in mlx5_common_dev_probe()
994 	if (!cdev) {  in mlx5_common_dev_probe()
996 		cdev = mlx5_common_dev_create(eal_dev, classes, mkvlist_p);  in mlx5_common_dev_probe()
997 		if (!cdev) {  in mlx5_common_dev_probe()
1004 		ret = mlx5_common_probe_again_args_validate(cdev, mkvlist_p);  in mlx5_common_dev_probe()
1017 	ret = is_valid_class_combination(classes | cdev->classes_loaded);  in mlx5_common_dev_probe()
1022 	ret = drivers_probe(cdev, classes, mkvlist_p);  in mlx5_common_dev_probe()
1041 		if (cdev->classes_loaded)  in mlx5_common_dev_probe()
1042 			drivers_remove(cdev, cdev->classes_loaded);  in mlx5_common_dev_probe()
1043 		mlx5_common_dev_release(cdev);  in mlx5_common_dev_probe()
1052 	struct mlx5_common_device *cdev;  in mlx5_common_dev_remove()  local
1055 	cdev = to_mlx5_device(eal_dev);  in mlx5_common_dev_remove()
1056 	if (!cdev)  in mlx5_common_dev_remove()
1059 	ret = drivers_remove(cdev, cdev->classes_loaded);  in mlx5_common_dev_remove()
1061 		mlx5_common_dev_release(cdev);  in mlx5_common_dev_remove()
1278 mlx5_devx_alloc_uar(struct mlx5_common_device *cdev)  in mlx5_devx_alloc_uar()  argument
1287 		uar_mapping = (cdev->config.dbnc == MLX5_SQ_DB_NCACHED) ?  in mlx5_devx_alloc_uar()
1296 		uar = mlx5_glue->devx_alloc_uar(cdev->ctx, uar_mapping);  in mlx5_devx_alloc_uar()
1307 			uar = mlx5_glue->devx_alloc_uar(cdev->ctx, uar_mapping);  in mlx5_devx_alloc_uar()
1315 			uar = mlx5_glue->devx_alloc_uar(cdev->ctx, uar_mapping);  in mlx5_devx_alloc_uar()
1356 mlx5_devx_uar_prepare(struct mlx5_common_device *cdev, struct mlx5_uar *uar)  in mlx5_devx_uar_prepare()  argument
1368 	uar_obj = mlx5_devx_alloc_uar(cdev);  in mlx5_devx_uar_prepare()