115 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,  in cdns3_get_dma_pos()  argument
120 	dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;  in cdns3_get_dma_pos()
163 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)  in cdns3_select_ep()  argument
165 	if (priv_dev->selected_ep == ep)  in cdns3_select_ep()
168 	priv_dev->selected_ep = ep;  in cdns3_select_ep()
169 	writel(ep, &priv_dev->regs->ep_sel);  in cdns3_select_ep()
179 static int cdns3_get_tdl(struct cdns3_device *priv_dev)  in cdns3_get_tdl()  argument
181 	if (priv_dev->dev_ver < DEV_VER_V3)  in cdns3_get_tdl()
182 		return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));  in cdns3_get_tdl()
184 		return readl(&priv_dev->regs->ep_tdl);  in cdns3_get_tdl()
197 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_free_trb_pool()  local
200 		dma_pool_free(priv_dev->eps_dma_pool,  in cdns3_free_trb_pool()
214 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_allocate_trb_pool()  local
223 		priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool,  in cdns3_allocate_trb_pool()
264 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_ep_stall_flush()  local
270 	       &priv_dev->regs->ep_cmd);  in cdns3_ep_stall_flush()
273 	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in cdns3_ep_stall_flush()
283 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)  in cdns3_hw_reset_eps_config()  argument
287 	writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);  in cdns3_hw_reset_eps_config()
289 	cdns3_allow_enable_l1(priv_dev, 0);  in cdns3_hw_reset_eps_config()
290 	priv_dev->hw_configured_flag = 0;  in cdns3_hw_reset_eps_config()
291 	priv_dev->onchip_used_size = 0;  in cdns3_hw_reset_eps_config()
292 	priv_dev->out_mem_is_allocated = 0;  in cdns3_hw_reset_eps_config()
293 	priv_dev->wait_for_setup = 0;  in cdns3_hw_reset_eps_config()
294 	priv_dev->using_streams = 0;  in cdns3_hw_reset_eps_config()
297 		if (priv_dev->eps[i])  in cdns3_hw_reset_eps_config()
298 			priv_dev->eps[i]->flags &= ~EP_CONFIGURED;  in cdns3_hw_reset_eps_config()
351 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)  in cdns3_allow_enable_l1()  argument
354 		writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);  in cdns3_allow_enable_l1()
356 		writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);  in cdns3_allow_enable_l1()
359 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)  in cdns3_get_speed()  argument
363 	reg = readl(&priv_dev->regs->usb_sts);  in cdns3_get_speed()
384 static int cdns3_start_all_request(struct cdns3_device *priv_dev,  in cdns3_start_all_request()  argument
404 			dev_dbg(priv_dev->dev, "Blocking external request\n");  in cdns3_start_all_request()
436 #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \  argument
508 static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,  in cdns3_wa2_gadget_giveback()  argument
524 		usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,  in cdns3_wa2_gadget_giveback()
532 			usb_gadget_map_request_by_dev(priv_dev->sysdev, req,  in cdns3_wa2_gadget_giveback()
541 		cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_wa2_gadget_giveback()
548 static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,  in cdns3_wa2_gadget_ep_queue()  argument
562 		cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);  in cdns3_wa2_gadget_ep_queue()
564 		reg = readl(&priv_dev->regs->ep_sts_en);  in cdns3_wa2_gadget_ep_queue()
567 		writel(reg, &priv_dev->regs->ep_sts_en);  in cdns3_wa2_gadget_ep_queue()
710 static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)  in cdns3_wa2_reset_tdl()  argument
712 	u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));  in cdns3_wa2_reset_tdl()
718 		       &priv_dev->regs->ep_cmd);  in cdns3_wa2_reset_tdl()
722 static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)  in cdns3_wa2_check_outq_status()  argument
727 	cdns3_select_ep(priv_dev, 0);  in cdns3_wa2_check_outq_status()
729 	ep_sts_reg = readl(&priv_dev->regs->ep_sts);  in cdns3_wa2_check_outq_status()
733 		struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];  in cdns3_wa2_check_outq_status()
746 				cdns3_select_ep(priv_dev, outq_ep->num |  in cdns3_wa2_check_outq_status()
748 				ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);  in cdns3_wa2_check_outq_status()
749 				ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);  in cdns3_wa2_check_outq_status()
752 				cdns3_set_register_bit(&priv_dev->regs->ep_cfg,  in cdns3_wa2_check_outq_status()
755 				cdns3_wa2_enable_detection(priv_dev, outq_ep,  in cdns3_wa2_check_outq_status()
758 				       &priv_dev->regs->ep_sts_en);  in cdns3_wa2_check_outq_status()
760 				cdns3_wa2_reset_tdl(priv_dev);  in cdns3_wa2_check_outq_status()
775 					       &priv_dev->regs->ep_cmd);  in cdns3_wa2_check_outq_status()
796 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_giveback()  local
805 		usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,  in cdns3_gadget_giveback()
811 		dma_sync_single_for_cpu(priv_dev->sysdev,  in cdns3_gadget_giveback()
824 	if (priv_dev->dev_ver < DEV_VER_V2) {  in cdns3_gadget_giveback()
825 		request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,  in cdns3_gadget_giveback()
835 	if (request->complete && request->buf != priv_dev->zlp_buf) {  in cdns3_gadget_giveback()
836 		spin_unlock(&priv_dev->lock);  in cdns3_gadget_giveback()
839 		spin_lock(&priv_dev->lock);  in cdns3_gadget_giveback()
842 	if (request->buf == priv_dev->zlp_buf)  in cdns3_gadget_giveback()
866 	struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,  in cdns3_free_aligned_request_buf()  local
871 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_free_aligned_request_buf()
873 	list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {  in cdns3_free_aligned_request_buf()
882 			spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_free_aligned_request_buf()
883 			dma_free_noncoherent(priv_dev->sysdev, buf->size,  in cdns3_free_aligned_request_buf()
886 			spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_free_aligned_request_buf()
890 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_free_aligned_request_buf()
896 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_prepare_aligned_request_buf()  local
914 		buf->buf = dma_alloc_noncoherent(priv_dev->sysdev,  in cdns3_prepare_aligned_request_buf()
928 				   &priv_dev->aligned_buf_wq);  in cdns3_prepare_aligned_request_buf()
935 			      &priv_dev->aligned_buf_list);  in cdns3_prepare_aligned_request_buf()
940 		dma_sync_single_for_cpu(priv_dev->sysdev,  in cdns3_prepare_aligned_request_buf()
947 	dma_sync_single_for_device(priv_dev->sysdev,  in cdns3_prepare_aligned_request_buf()
959 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_wa1_update_guard()  local
964 		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);  in cdns3_wa1_update_guard()
978 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,  in cdns3_wa1_tray_restore_cycle_bit()  argument
984 	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);  in cdns3_wa1_tray_restore_cycle_bit()
985 	dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);  in cdns3_wa1_tray_restore_cycle_bit()
994 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_ep_run_stream_transfer()  local
1043 	if (priv_dev->dev_ver >= DEV_VER_V2) {  in cdns3_ep_run_stream_transfer()
1044 		if (priv_dev->gadget.speed == USB_SPEED_SUPER)  in cdns3_ep_run_stream_transfer()
1061 	       &priv_dev->regs->ep_traddr);  in cdns3_ep_run_stream_transfer()
1066 		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);  in cdns3_ep_run_stream_transfer()
1075 		if (priv_dev->dev_ver < DEV_VER_V2)  in cdns3_ep_run_stream_transfer()
1077 			       &priv_dev->regs->ep_cmd);  in cdns3_ep_run_stream_transfer()
1078 		else if (priv_dev->dev_ver > DEV_VER_V2)  in cdns3_ep_run_stream_transfer()
1079 			writel(tdl, &priv_dev->regs->ep_tdl);  in cdns3_ep_run_stream_transfer()
1082 		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);  in cdns3_ep_run_stream_transfer()
1084 		       EP_CMD_ERDY, &priv_dev->regs->ep_cmd);  in cdns3_ep_run_stream_transfer()
1087 					 readl(&priv_dev->regs->ep_traddr));  in cdns3_ep_run_stream_transfer()
1091 	__cdns3_gadget_wakeup(priv_dev);  in cdns3_ep_run_stream_transfer()
1098 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_rearm_drdy_if_needed()  local
1100 	if (priv_dev->dev_ver < DEV_VER_V3)  in cdns3_rearm_drdy_if_needed()
1103 	if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) {  in cdns3_rearm_drdy_if_needed()
1104 		writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts);  in cdns3_rearm_drdy_if_needed()
1105 		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);  in cdns3_rearm_drdy_if_needed()
1119 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_ep_run_transfer()  local
1167 		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);  in cdns3_ep_run_transfer()
1168 		dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);  in cdns3_ep_run_transfer()
1214 	if (priv_dev->dev_ver <= DEV_VER_V2)  in cdns3_ep_run_transfer()
1220 	if (priv_dev->dev_ver >= DEV_VER_V2) {  in cdns3_ep_run_transfer()
1225 		if (priv_dev->gadget.speed == USB_SPEED_SUPER)  in cdns3_ep_run_transfer()
1275 		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_dev->dev_ver <= DEV_VER_V2)  in cdns3_ep_run_transfer()
1329 	if (priv_dev->dev_ver < DEV_VER_V2 &&  in cdns3_ep_run_transfer()
1332 		u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));  in cdns3_ep_run_transfer()
1342 			       &priv_dev->regs->ep_cmd);  in cdns3_ep_run_transfer()
1355 	if (priv_dev->dev_ver <= DEV_VER_V2)  in cdns3_ep_run_transfer()
1356 		cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);  in cdns3_ep_run_transfer()
1394 			cdns3_set_register_bit(&priv_dev->regs->ep_cfg,  in cdns3_ep_run_transfer()
1400 					&priv_dev->regs->ep_traddr);  in cdns3_ep_run_transfer()
1408 		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);  in cdns3_ep_run_transfer()
1409 		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);  in cdns3_ep_run_transfer()
1412 					 readl(&priv_dev->regs->ep_traddr));  in cdns3_ep_run_transfer()
1416 	__cdns3_gadget_wakeup(priv_dev);  in cdns3_ep_run_transfer()
1421 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)  in cdns3_set_hw_configuration()  argument
1426 	if (priv_dev->hw_configured_flag)  in cdns3_set_hw_configuration()
1429 	writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);  in cdns3_set_hw_configuration()
1431 	cdns3_set_register_bit(&priv_dev->regs->usb_conf,  in cdns3_set_hw_configuration()
1434 	priv_dev->hw_configured_flag = 1;  in cdns3_set_hw_configuration()
1436 	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {  in cdns3_set_hw_configuration()
1439 			cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_set_hw_configuration()
1443 	cdns3_allow_enable_l1(priv_dev, 1);  in cdns3_set_hw_configuration()
1488 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_trb_handled()  local
1494 	current_index = cdns3_get_dma_pos(priv_dev, priv_ep);  in cdns3_trb_handled()
1495 	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);  in cdns3_trb_handled()
1548 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,  in cdns3_transfer_completed()  argument
1567 			if (priv_ep->dequeue == cdns3_get_dma_pos(priv_dev, priv_ep) &&  in cdns3_transfer_completed()
1580 			cdns3_select_ep(priv_dev, priv_ep->endpoint.address);  in cdns3_transfer_completed()
1621 			cdns3_select_ep(priv_dev, priv_ep->endpoint.address);  in cdns3_transfer_completed()
1627 				dev_warn(priv_dev->dev,  in cdns3_transfer_completed()
1649 		cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_transfer_completed()
1654 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_rearm_transfer()  local
1663 		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);  in cdns3_rearm_transfer()
1665 		__cdns3_gadget_wakeup(priv_dev);  in cdns3_rearm_transfer()
1668 					 readl(&priv_dev->regs->ep_traddr));  in cdns3_rearm_transfer()
1675 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_reprogram_tdl()  local
1684 	writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);  in cdns3_reprogram_tdl()
1695 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_check_ep_interrupt_proceed()  local
1701 	cdns3_select_ep(priv_dev, priv_ep->endpoint.address);  in cdns3_check_ep_interrupt_proceed()
1703 	trace_cdns3_epx_irq(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1705 	ep_sts_reg = readl(&priv_dev->regs->ep_sts);  in cdns3_check_ep_interrupt_proceed()
1706 	writel(ep_sts_reg, &priv_dev->regs->ep_sts);  in cdns3_check_ep_interrupt_proceed()
1711 		tdl = cdns3_get_tdl(priv_dev);  in cdns3_check_ep_interrupt_proceed()
1724 			       &priv_dev->regs->ep_cmd);  in cdns3_check_ep_interrupt_proceed()
1733 				cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1741 		    priv_dev->dev_ver < DEV_VER_V2)) {  in cdns3_check_ep_interrupt_proceed()
1755 				u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);  in cdns3_check_ep_interrupt_proceed()
1758 				writel(ep_cfg, &priv_dev->regs->ep_cfg);  in cdns3_check_ep_interrupt_proceed()
1762 			cdns3_transfer_completed(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1767 				cdns3_start_all_request(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1787 				cdns3_transfer_completed(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1796 			cdns3_transfer_completed(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1807 		cdns3_transfer_completed(priv_dev, priv_ep);  in cdns3_check_ep_interrupt_proceed()
1816 	if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&  in cdns3_check_ep_interrupt_proceed()
1823 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)  in cdns3_disconnect_gadget()  argument
1825 	if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect)  in cdns3_disconnect_gadget()
1826 		priv_dev->gadget_driver->disconnect(&priv_dev->gadget);  in cdns3_disconnect_gadget()
1835 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,  in cdns3_check_usb_interrupt_proceed()  argument
1837 __must_hold(&priv_dev->lock)  in cdns3_check_usb_interrupt_proceed()
1841 	trace_cdns3_usb_irq(priv_dev, usb_ists);  in cdns3_check_usb_interrupt_proceed()
1848 		if (readl(&priv_dev->regs->drbl))  in cdns3_check_usb_interrupt_proceed()
1849 			__cdns3_gadget_wakeup(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1854 		speed = cdns3_get_speed(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1855 		priv_dev->gadget.speed = speed;  in cdns3_check_usb_interrupt_proceed()
1856 		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);  in cdns3_check_usb_interrupt_proceed()
1857 		cdns3_ep0_config(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1862 		spin_unlock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1863 		cdns3_disconnect_gadget(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1864 		spin_lock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1865 		priv_dev->gadget.speed = USB_SPEED_UNKNOWN;  in cdns3_check_usb_interrupt_proceed()
1866 		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);  in cdns3_check_usb_interrupt_proceed()
1867 		cdns3_hw_reset_eps_config(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1871 		if (priv_dev->gadget_driver &&  in cdns3_check_usb_interrupt_proceed()
1872 		    priv_dev->gadget_driver->suspend) {  in cdns3_check_usb_interrupt_proceed()
1873 			spin_unlock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1874 			priv_dev->gadget_driver->suspend(&priv_dev->gadget);  in cdns3_check_usb_interrupt_proceed()
1875 			spin_lock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1880 		if (priv_dev->gadget_driver &&  in cdns3_check_usb_interrupt_proceed()
1881 		    priv_dev->gadget_driver->resume) {  in cdns3_check_usb_interrupt_proceed()
1882 			spin_unlock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1883 			priv_dev->gadget_driver->resume(&priv_dev->gadget);  in cdns3_check_usb_interrupt_proceed()
1884 			spin_lock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1890 		if (priv_dev->gadget_driver) {  in cdns3_check_usb_interrupt_proceed()
1891 			spin_unlock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1892 			usb_gadget_udc_reset(&priv_dev->gadget,  in cdns3_check_usb_interrupt_proceed()
1893 					     priv_dev->gadget_driver);  in cdns3_check_usb_interrupt_proceed()
1894 			spin_lock(&priv_dev->lock);  in cdns3_check_usb_interrupt_proceed()
1897 			speed = cdns3_get_speed(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1898 			priv_dev->gadget.speed = speed;  in cdns3_check_usb_interrupt_proceed()
1899 			cdns3_hw_reset_eps_config(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1900 			cdns3_ep0_config(priv_dev);  in cdns3_check_usb_interrupt_proceed()
1915 	struct cdns3_device *priv_dev = data;  in cdns3_device_irq_handler()  local
1916 	struct cdns *cdns = dev_get_drvdata(priv_dev->dev);  in cdns3_device_irq_handler()
1924 	reg = readl(&priv_dev->regs->usb_ists);  in cdns3_device_irq_handler()
1933 		reg = ~reg & readl(&priv_dev->regs->usb_ien);  in cdns3_device_irq_handler()
1935 		writel(reg, &priv_dev->regs->usb_ien);  in cdns3_device_irq_handler()
1940 	reg = readl(&priv_dev->regs->ep_ists);  in cdns3_device_irq_handler()
1942 		writel(0, &priv_dev->regs->ep_ien);  in cdns3_device_irq_handler()
1960 	struct cdns3_device *priv_dev = data;  in cdns3_device_thread_irq_handler()  local
1967 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_device_thread_irq_handler()
1969 	reg = readl(&priv_dev->regs->usb_ists);  in cdns3_device_thread_irq_handler()
1971 		writel(reg, &priv_dev->regs->usb_ists);  in cdns3_device_thread_irq_handler()
1972 		writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);  in cdns3_device_thread_irq_handler()
1973 		cdns3_check_usb_interrupt_proceed(priv_dev, reg);  in cdns3_device_thread_irq_handler()
1977 	reg = readl(&priv_dev->regs->ep_ists);  in cdns3_device_thread_irq_handler()
1981 		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);  in cdns3_device_thread_irq_handler()
1987 		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);  in cdns3_device_thread_irq_handler()
1998 		cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);  in cdns3_device_thread_irq_handler()
2002 	if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)  in cdns3_device_thread_irq_handler()
2003 		cdns3_wa2_check_outq_status(priv_dev);  in cdns3_device_thread_irq_handler()
2006 	writel(~0, &priv_dev->regs->ep_ien);  in cdns3_device_thread_irq_handler()
2007 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_device_thread_irq_handler()
2025 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,  in cdns3_ep_onchip_buffer_reserve()  argument
2031 	remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;  in cdns3_ep_onchip_buffer_reserve()
2037 		priv_dev->onchip_used_size += size;  in cdns3_ep_onchip_buffer_reserve()
2045 		if (priv_dev->out_mem_is_allocated >= size)  in cdns3_ep_onchip_buffer_reserve()
2048 		required = size - priv_dev->out_mem_is_allocated;  in cdns3_ep_onchip_buffer_reserve()
2053 		priv_dev->out_mem_is_allocated += required;  in cdns3_ep_onchip_buffer_reserve()
2054 		priv_dev->onchip_used_size += required;  in cdns3_ep_onchip_buffer_reserve()
2060 static void cdns3_configure_dmult(struct cdns3_device *priv_dev,  in cdns3_configure_dmult()  argument
2063 	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;  in cdns3_configure_dmult()
2066 	if (priv_dev->dev_ver <= DEV_VER_V2)  in cdns3_configure_dmult()
2069 	if (priv_dev->dev_ver == DEV_VER_V2)  in cdns3_configure_dmult()
2072 	if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {  in cdns3_configure_dmult()
2102 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_ep_config()  local
2110 	buffering = priv_dev->ep_buf_size - 1;  in cdns3_ep_config()
2112 	cdns3_configure_dmult(priv_dev, priv_ep);  in cdns3_ep_config()
2118 		if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)  in cdns3_ep_config()
2124 		if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)  in cdns3_ep_config()
2132 	switch (priv_dev->gadget.speed) {  in cdns3_ep_config()
2142 			maxburst = priv_dev->ep_buf_size - 1;  in cdns3_ep_config()
2167 	if (priv_dev->dev_ver < DEV_VER_V2)  in cdns3_ep_config()
2174 	if (!priv_dev->hw_configured_flag) {  in cdns3_ep_config()
2175 		ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,  in cdns3_ep_config()
2178 			dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");  in cdns3_ep_config()
2186 	if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {  in cdns3_ep_config()
2187 		if (priv_dev->dev_ver >= DEV_VER_V3) {  in cdns3_ep_config()
2194 			cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb,  in cdns3_ep_config()
2207 	cdns3_select_ep(priv_dev, bEndpointAddress);  in cdns3_ep_config()
2208 	writel(ep_cfg, &priv_dev->regs->ep_cfg);  in cdns3_ep_config()
2211 	dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",  in cdns3_ep_config()
2226 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,  in cdns3_find_available_ep()  argument
2232 	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {  in cdns3_find_available_ep()
2275 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_match_ep()  local
2279 	priv_ep = cdns3_find_available_ep(priv_dev, desc);  in cdns3_gadget_match_ep()
2281 		dev_err(priv_dev->dev, "no available ep\n");  in cdns3_gadget_match_ep()
2285 	dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);  in cdns3_gadget_match_ep()
2287 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_match_ep()
2301 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_match_ep()
2356 	struct cdns3_device *priv_dev;  in cdns3_gadget_ep_enable()  local
2371 	priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_enable()
2375 		dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");  in cdns3_gadget_ep_enable()
2380 		dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");  in cdns3_gadget_ep_enable()
2384 	if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,  in cdns3_gadget_ep_enable()
2388 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_enable()
2396 		dev_err(priv_dev->dev, "Driver is limited to %d period\n",  in cdns3_gadget_ep_enable()
2404 	cdns3_select_ep(priv_dev, bEndpointAddress);  in cdns3_gadget_ep_enable()
2422 		if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {  in cdns3_gadget_ep_enable()
2428 			priv_dev->using_streams |= true;  in cdns3_gadget_ep_enable()
2442 	cdns3_select_ep(priv_dev, bEndpointAddress);  in cdns3_gadget_ep_enable()
2446 	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);  in cdns3_gadget_ep_enable()
2448 	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in cdns3_gadget_ep_enable()
2459 	cdns3_set_register_bit(&priv_dev->regs->ep_ien,  in cdns3_gadget_ep_enable()
2462 	if (priv_dev->dev_ver < DEV_VER_V2)  in cdns3_gadget_ep_enable()
2463 		cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);  in cdns3_gadget_ep_enable()
2465 	writel(reg, &priv_dev->regs->ep_sts_en);  in cdns3_gadget_ep_enable()
2474 	reg = readl(&priv_dev->regs->ep_sts);  in cdns3_gadget_ep_enable()
2480 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_enable()
2495 	struct cdns3_device *priv_dev;  in cdns3_gadget_ep_disable()  local
2508 	priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_disable()
2510 	if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),  in cdns3_gadget_ep_disable()
2514 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_disable()
2518 	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);  in cdns3_gadget_ep_disable()
2520 	ep_cfg = readl(&priv_dev->regs->ep_cfg);  in cdns3_gadget_ep_disable()
2522 	writel(ep_cfg, &priv_dev->regs->ep_cfg);  in cdns3_gadget_ep_disable()
2529 	readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,  in cdns3_gadget_ep_disable()
2531 	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);  in cdns3_gadget_ep_disable()
2533 	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in cdns3_gadget_ep_disable()
2537 		dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",  in cdns3_gadget_ep_disable()
2570 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_disable()
2588 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in __cdns3_gadget_ep_queue()  local
2597 	if (priv_dev->dev_ver < DEV_VER_V2) {  in __cdns3_gadget_ep_queue()
2598 		ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,  in __cdns3_gadget_ep_queue()
2610 		ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,  in __cdns3_gadget_ep_queue()
2626 		if (priv_dev->hw_configured_flag &&  in __cdns3_gadget_ep_queue()
2629 			cdns3_start_all_request(priv_dev, priv_ep);  in __cdns3_gadget_ep_queue()
2631 		if (priv_dev->hw_configured_flag && priv_ep->prime_flag)  in __cdns3_gadget_ep_queue()
2632 			cdns3_start_all_request(priv_dev, priv_ep);  in __cdns3_gadget_ep_queue()
2643 	struct cdns3_device *priv_dev;  in cdns3_gadget_ep_queue()  local
2651 	priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_queue()
2653 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_queue()
2662 		zlp_request->buf = priv_dev->zlp_buf;  in cdns3_gadget_ep_queue()
2668 		dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",  in cdns3_gadget_ep_queue()
2673 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_queue()
2688 	struct cdns3_device *priv_dev;  in cdns3_gadget_ep_dequeue()  local
2700 	priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_dequeue()
2702 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_dequeue()
2708 	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);  in cdns3_gadget_ep_dequeue()
2732 		writel(EP_CMD_DFLUSH, &priv_dev->regs->ep_cmd);  in cdns3_gadget_ep_dequeue()
2735 		readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in cdns3_gadget_ep_dequeue()
2754 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_dequeue()
2765 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in __cdns3_gadget_ep_set_halt()  local
2770 		u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);  in __cdns3_gadget_ep_set_halt()
2786 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in __cdns3_gadget_ep_clear_halt()  local
2806 	writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);  in __cdns3_gadget_ep_clear_halt()
2809 	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in __cdns3_gadget_ep_clear_halt()
2823 	cdns3_start_all_request(priv_dev, priv_ep);  in __cdns3_gadget_ep_clear_halt()
2837 	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;  in cdns3_gadget_ep_set_halt()  local
2844 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_ep_set_halt()
2846 	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);  in cdns3_gadget_ep_set_halt()
2855 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_ep_set_halt()
2881 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_get_frame()  local
2883 	return readl(&priv_dev->regs->usb_itpn);  in cdns3_gadget_get_frame()
2886 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)  in __cdns3_gadget_wakeup()  argument
2890 	speed = cdns3_get_speed(priv_dev);  in __cdns3_gadget_wakeup()
2896 	writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);  in __cdns3_gadget_wakeup()
2903 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_wakeup()  local
2907 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_wakeup()
2908 	ret = __cdns3_gadget_wakeup(priv_dev);  in cdns3_gadget_wakeup()
2909 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_wakeup()
2916 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_set_selfpowered()  local
2919 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_set_selfpowered()
2920 	priv_dev->is_selfpowered = !!is_selfpowered;  in cdns3_gadget_set_selfpowered()
2921 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_set_selfpowered()
2927 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_pullup()  local
2930 		writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);  in cdns3_gadget_pullup()
2932 		writel(~0, &priv_dev->regs->ep_ists);  in cdns3_gadget_pullup()
2933 		writel(~0, &priv_dev->regs->usb_ists);  in cdns3_gadget_pullup()
2934 		writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);  in cdns3_gadget_pullup()
2940 static void cdns3_gadget_config(struct cdns3_device *priv_dev)  in cdns3_gadget_config()  argument
2942 	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;  in cdns3_gadget_config()
2945 	cdns3_ep0_config(priv_dev);  in cdns3_gadget_config()
2954 	if (priv_dev->dev_ver == DEV_VER_TI_V1) {  in cdns3_gadget_config()
2977 	writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);  in cdns3_gadget_config()
2979 	cdns3_configure_dmult(priv_dev, NULL);  in cdns3_gadget_config()
2992 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_udc_start()  local
2996 	spin_lock_irqsave(&priv_dev->lock, flags);  in cdns3_gadget_udc_start()
2997 	priv_dev->gadget_driver = driver;  in cdns3_gadget_udc_start()
3004 		writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);  in cdns3_gadget_udc_start()
3005 		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);  in cdns3_gadget_udc_start()
3008 		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);  in cdns3_gadget_udc_start()
3013 		dev_err(priv_dev->dev,  in cdns3_gadget_udc_start()
3023 	cdns3_gadget_config(priv_dev);  in cdns3_gadget_udc_start()
3024 	spin_unlock_irqrestore(&priv_dev->lock, flags);  in cdns3_gadget_udc_start()
3036 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_udc_stop()  local
3042 	priv_dev->gadget_driver = NULL;  in cdns3_gadget_udc_stop()
3044 	priv_dev->onchip_used_size = 0;  in cdns3_gadget_udc_stop()
3045 	priv_dev->out_mem_is_allocated = 0;  in cdns3_gadget_udc_stop()
3046 	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;  in cdns3_gadget_udc_stop()
3048 	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {  in cdns3_gadget_udc_stop()
3051 		cdns3_select_ep(priv_dev, bEndpointAddress);  in cdns3_gadget_udc_stop()
3052 		writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);  in cdns3_gadget_udc_stop()
3053 		readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,  in cdns3_gadget_udc_stop()
3060 	writel(0, &priv_dev->regs->usb_ien);  in cdns3_gadget_udc_stop()
3061 	writel(0, &priv_dev->regs->usb_pwr);  in cdns3_gadget_udc_stop()
3062 	writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);  in cdns3_gadget_udc_stop()
3079 	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);  in cdns3_gadget_check_config()  local
3113 	if (total > priv_dev->onchip_buffers)  in cdns3_gadget_check_config()
3116 	priv_dev->ep_buf_size = (priv_dev->onchip_buffers - 2 - iso) / (n_in + out);  in cdns3_gadget_check_config()
3132 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)  in cdns3_free_all_eps()  argument
3137 	priv_dev->eps[16] = NULL;  in cdns3_free_all_eps()
3140 		if (priv_dev->eps[i]) {  in cdns3_free_all_eps()
3141 			cdns3_free_trb_pool(priv_dev->eps[i]);  in cdns3_free_all_eps()
3142 			devm_kfree(priv_dev->dev, priv_dev->eps[i]);  in cdns3_free_all_eps()
3152 static int cdns3_init_eps(struct cdns3_device *priv_dev)  in cdns3_init_eps()  argument
3162 	ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);  in cdns3_init_eps()
3163 	iso_ep_reg = readl(&priv_dev->regs->usb_cap4);  in cdns3_init_eps()
3165 	dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");  in cdns3_init_eps()
3176 			priv_dev->eps[i] = priv_dev->eps[0];  in cdns3_init_eps()
3180 		priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),  in cdns3_init_eps()
3186 		priv_ep->cdns3_dev = priv_dev;  in cdns3_init_eps()
3187 		priv_dev->eps[i] = priv_ep;  in cdns3_init_eps()
3192 			ret = cdns3_init_ep0(priv_dev, priv_ep);  in cdns3_init_eps()
3194 				dev_err(priv_dev->dev, "Failed to init ep0\n");  in cdns3_init_eps()
3218 				      &priv_dev->gadget.ep_list);  in cdns3_init_eps()
3223 		dev_dbg(priv_dev->dev, "Initialized  %s support: %s %s\n",  in cdns3_init_eps()
3235 	cdns3_free_all_eps(priv_dev);  in cdns3_init_eps()
3241 	struct cdns3_device *priv_dev = container_of(dev,  in cdns3_gadget_release()  local
3244 	kfree(priv_dev);  in cdns3_gadget_release()
3249 	struct cdns3_device *priv_dev;  in cdns3_gadget_exit()  local
3251 	priv_dev = cdns->gadget_dev;  in cdns3_gadget_exit()
3257 	usb_del_gadget(&priv_dev->gadget);  in cdns3_gadget_exit()
3258 	devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);  in cdns3_gadget_exit()
3260 	cdns3_free_all_eps(priv_dev);  in cdns3_gadget_exit()
3262 	while (!list_empty(&priv_dev->aligned_buf_list)) {  in cdns3_gadget_exit()
3265 		buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);  in cdns3_gadget_exit()
3266 		dma_free_noncoherent(priv_dev->sysdev, buf->size,  in cdns3_gadget_exit()
3275 	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,  in cdns3_gadget_exit()
3276 			  priv_dev->setup_dma);  in cdns3_gadget_exit()
3277 	dma_pool_destroy(priv_dev->eps_dma_pool);  in cdns3_gadget_exit()
3279 	kfree(priv_dev->zlp_buf);  in cdns3_gadget_exit()
3280 	usb_put_gadget(&priv_dev->gadget);  in cdns3_gadget_exit()
3287 	struct cdns3_device *priv_dev;  in cdns3_gadget_start()  local
3291 	priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);  in cdns3_gadget_start()
3292 	if (!priv_dev)  in cdns3_gadget_start()
3295 	usb_initialize_gadget(cdns->dev, &priv_dev->gadget,  in cdns3_gadget_start()
3297 	cdns->gadget_dev = priv_dev;  in cdns3_gadget_start()
3298 	priv_dev->sysdev = cdns->dev;  in cdns3_gadget_start()
3299 	priv_dev->dev = cdns->dev;  in cdns3_gadget_start()
3300 	priv_dev->regs = cdns->dev_regs;  in cdns3_gadget_start()
3302 	device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",  in cdns3_gadget_start()
3303 				 &priv_dev->onchip_buffers);  in cdns3_gadget_start()
3305 	if (priv_dev->onchip_buffers <=  0) {  in cdns3_gadget_start()
3306 		u32 reg = readl(&priv_dev->regs->usb_cap2);  in cdns3_gadget_start()
3308 		priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);  in cdns3_gadget_start()
3311 	if (!priv_dev->onchip_buffers)  in cdns3_gadget_start()
3312 		priv_dev->onchip_buffers = 256;  in cdns3_gadget_start()
3333 	priv_dev->gadget.max_speed = max_speed;  in cdns3_gadget_start()
3334 	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;  in cdns3_gadget_start()
3335 	priv_dev->gadget.ops = &cdns3_gadget_ops;  in cdns3_gadget_start()
3336 	priv_dev->gadget.name = "usb-ss-gadget";  in cdns3_gadget_start()
3337 	priv_dev->gadget.quirk_avoids_skb_reserve = 1;  in cdns3_gadget_start()
3338 	priv_dev->gadget.irq = cdns->dev_irq;  in cdns3_gadget_start()
3340 	spin_lock_init(&priv_dev->lock);  in cdns3_gadget_start()
3341 	INIT_WORK(&priv_dev->pending_status_wq,  in cdns3_gadget_start()
3344 	INIT_WORK(&priv_dev->aligned_buf_wq,  in cdns3_gadget_start()
3348 	INIT_LIST_HEAD(&priv_dev->gadget.ep_list);  in cdns3_gadget_start()
3349 	INIT_LIST_HEAD(&priv_dev->aligned_buf_list);  in cdns3_gadget_start()
3350 	priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool",  in cdns3_gadget_start()
3351 						 priv_dev->sysdev,  in cdns3_gadget_start()
3353 	if (!priv_dev->eps_dma_pool) {  in cdns3_gadget_start()
3354 		dev_err(priv_dev->dev, "Failed to create TRB dma pool\n");  in cdns3_gadget_start()
3359 	ret = cdns3_init_eps(priv_dev);  in cdns3_gadget_start()
3361 		dev_err(priv_dev->dev, "Failed to create endpoints\n");  in cdns3_gadget_start()
3366 	priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,  in cdns3_gadget_start()
3367 						 &priv_dev->setup_dma, GFP_DMA);  in cdns3_gadget_start()
3368 	if (!priv_dev->setup_buf) {  in cdns3_gadget_start()
3373 	priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);  in cdns3_gadget_start()
3375 	dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",  in cdns3_gadget_start()
3376 		readl(&priv_dev->regs->usb_cap6));  in cdns3_gadget_start()
3377 	dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",  in cdns3_gadget_start()
3378 		readl(&priv_dev->regs->usb_cap1));  in cdns3_gadget_start()
3379 	dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",  in cdns3_gadget_start()
3380 		readl(&priv_dev->regs->usb_cap2));  in cdns3_gadget_start()
3382 	priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);  in cdns3_gadget_start()
3383 	if (priv_dev->dev_ver >= DEV_VER_V2)  in cdns3_gadget_start()
3384 		priv_dev->gadget.sg_supported = 1;  in cdns3_gadget_start()
3386 	priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);  in cdns3_gadget_start()
3387 	if (!priv_dev->zlp_buf) {  in cdns3_gadget_start()
3393 	ret = usb_add_gadget(&priv_dev->gadget);  in cdns3_gadget_start()
3395 		dev_err(priv_dev->dev, "Failed to add gadget\n");  in cdns3_gadget_start()
3401 	kfree(priv_dev->zlp_buf);  in cdns3_gadget_start()
3403 	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,  in cdns3_gadget_start()
3404 			  priv_dev->setup_dma);  in cdns3_gadget_start()
3406 	cdns3_free_all_eps(priv_dev);  in cdns3_gadget_start()
3408 	dma_pool_destroy(priv_dev->eps_dma_pool);  in cdns3_gadget_start()
3410 	usb_put_gadget(&priv_dev->gadget);  in cdns3_gadget_start()
3457 	struct cdns3_device *priv_dev = cdns->gadget_dev;  in cdns3_gadget_suspend()  local
3460 	cdns3_disconnect_gadget(priv_dev);  in cdns3_gadget_suspend()
3463 	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;  in cdns3_gadget_suspend()
3464 	usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);  in cdns3_gadget_suspend()
3465 	cdns3_hw_reset_eps_config(priv_dev);  in cdns3_gadget_suspend()
3468 	writel(0, &priv_dev->regs->usb_ien);  in cdns3_gadget_suspend()
3475 	struct cdns3_device *priv_dev = cdns->gadget_dev;  in cdns3_gadget_resume()  local
3477 	if (!priv_dev->gadget_driver)  in cdns3_gadget_resume()
3480 	cdns3_gadget_config(priv_dev);  in cdns3_gadget_resume()
3482 		writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);  in cdns3_gadget_resume()