| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * Cadence USBSS DRD Driver - gadget side. |
| * |
| * Copyright (C) 2018-2019 Cadence Design Systems. |
| * Copyright (C) 2017-2018 NXP |
| * |
| * Authors: Pawel Jez <pjez@cadence.com>, |
| * Pawel Laszczak <pawell@cadence.com> |
| * Peter Chen <peter.chen@nxp.com> |
| */ |
| |
| /* |
| * Work around 1: |
| * At some situations, the controller may get stale data address in TRB |
| * at below sequences: |
| * 1. Controller read TRB includes data address |
| * 2. Software updates TRBs includes data address and Cycle bit |
| * 3. Controller read TRB which includes Cycle bit |
| * 4. DMA run with stale data address |
| * |
| * To fix this problem, driver needs to make the first TRB in TD as invalid. |
| * After preparing all TRBs driver needs to check the position of DMA and |
| * if the DMA point to the first just added TRB and doorbell is 1, |
| * then driver must defer making this TRB as valid. This TRB will be make |
| * as valid during adding next TRB only if DMA is stopped or at TRBERR |
| * interrupt. |
| * |
| * Issue has been fixed in DEV_VER_V3 version of controller. |
| * |
| * Work around 2: |
| * Controller for OUT endpoints has shared on-chip buffers for all incoming |
| * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA |
| * in correct order. If the first packet in the buffer will not be handled, |
| * then the following packets directed for other endpoints and functions |
| * will be blocked. |
| * Additionally the packets directed to one endpoint can block entire on-chip |
| * buffers. In this case transfer to other endpoints also will blocked. |
| * |
| * To resolve this issue after raising the descriptor missing interrupt |
| * driver prepares internal usb_request object and use it to arm DMA transfer. |
| * |
| * The problematic situation was observed in case when endpoint has been enabled |
| * but no usb_request were queued. Driver try detects such endpoints and will |
| * use this workaround only for these endpoint. |
| * |
| * Driver use limited number of buffer. This number can be set by macro |
| * CDNS3_WA2_NUM_BUFFERS. |
| * |
| * Such blocking situation was observed on ACM gadget. For this function |
| * host send OUT data packet but ACM function is not prepared for this packet. |
| * It's cause that buffer placed in on chip memory block transfer to other |
| * endpoints. |
| * |
| * Issue has been fixed in DEV_VER_V2 version of controller. |
| * |
| */ |
| |
| #include <linux/dma-mapping.h> |
| #include <linux/usb/gadget.h> |
| #include <linux/module.h> |
| #include <linux/dmapool.h> |
| #include <linux/iopoll.h> |
| |
| #include "core.h" |
| #include "gadget-export.h" |
| #include "cdns3-gadget.h" |
| #include "cdns3-trace.h" |
| #include "drd.h" |
| |
| static int __cdns3_gadget_ep_queue(struct usb_ep *ep, |
| struct usb_request *request, |
| gfp_t gfp_flags); |
| |
| static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, |
| struct usb_request *request); |
| |
| static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep, |
| struct usb_request *request); |
| |
| /** |
| * cdns3_clear_register_bit - clear bit in given register. |
| * @ptr: address of device controller register to be read and changed |
| * @mask: bits requested to clar |
| */ |
| static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask) |
| { |
| mask = readl(ptr) & ~mask; |
| writel(mask, ptr); |
| } |
| |
| /** |
| * cdns3_set_register_bit - set bit in given register. |
| * @ptr: address of device controller register to be read and changed |
| * @mask: bits requested to set |
| */ |
| void cdns3_set_register_bit(void __iomem *ptr, u32 mask) |
| { |
| mask = readl(ptr) | mask; |
| writel(mask, ptr); |
| } |
| |
| /** |
| * cdns3_ep_addr_to_index - Macro converts endpoint address to |
| * index of endpoint object in cdns3_device.eps[] container |
| * @ep_addr: endpoint address for which endpoint object is required |
| * |
| */ |
| u8 cdns3_ep_addr_to_index(u8 ep_addr) |
| { |
| return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0)); |
| } |
| |
| static int cdns3_get_dma_pos(struct cdns3_device *priv_dev, |
| struct cdns3_endpoint *priv_ep) |
| { |
| int dma_index; |
| |
| dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma; |
| |
| return dma_index / TRB_SIZE; |
| } |
| |
| /** |
| * cdns3_next_request - returns next request from list |
| * @list: list containing requests |
| * |
| * Returns request or NULL if no requests in list |
| */ |
| struct usb_request *cdns3_next_request(struct list_head *list) |
| { |
| return list_first_entry_or_null(list, struct usb_request, list); |
| } |
| |
| /** |
| * cdns3_next_align_buf - returns next buffer from list |
| * @list: list containing buffers |
| * |
| * Returns buffer or NULL if no buffers in list |
| */ |
| static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list) |
| { |
| return list_first_entry_or_null(list, struct cdns3_aligned_buf, list); |
| } |
| |
| /** |
| * cdns3_next_priv_request - returns next request from list |
| * @list: list containing requests |
| * |
| * Returns request or NULL if no requests in list |
| */ |
| static struct cdns3_request *cdns3_next_priv_request(struct list_head *list) |
| { |
| return list_first_entry_or_null(list, struct cdns3_request, list); |
| } |
| |
| /** |
| * cdns3_select_ep - selects endpoint |
| * @priv_dev: extended gadget object |
| * @ep: endpoint address |
| */ |
| void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep) |
| { |
| if (priv_dev->selected_ep == ep) |
| return; |
| |
| priv_dev->selected_ep = ep; |
| writel(ep, &priv_dev->regs->ep_sel); |
| } |
| |
| /** |
| * cdns3_get_tdl - gets current tdl for selected endpoint. |
| * @priv_dev: extended gadget object |
| * |
| * Before calling this function the appropriate endpoint must |
| * be selected by means of cdns3_select_ep function. |
| */ |
| static int cdns3_get_tdl(struct cdns3_device *priv_dev) |
| { |
| if (priv_dev->dev_ver < DEV_VER_V3) |
| return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); |
| else |
| return readl(&priv_dev->regs->ep_tdl); |
| } |
| |
| dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep, |
| struct cdns3_trb *trb) |
| { |
| u32 offset = (char *)trb - (char *)priv_ep->trb_pool; |
| |
| return priv_ep->trb_pool_dma + offset; |
| } |
| |
| static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| |
| if (priv_ep->trb_pool) { |
| dma_pool_free(priv_dev->eps_dma_pool, |
| priv_ep->trb_pool, priv_ep->trb_pool_dma); |
| priv_ep->trb_pool = NULL; |
| } |
| } |
| |
| /** |
| * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint |
| * @priv_ep: endpoint object |
| * |
| * Function will return 0 on success or -ENOMEM on allocation error |
| */ |
| int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| int ring_size = TRB_RING_SIZE; |
| int num_trbs = ring_size / TRB_SIZE; |
| struct cdns3_trb *link_trb; |
| |
| if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size) |
| cdns3_free_trb_pool(priv_ep); |
| |
| if (!priv_ep->trb_pool) { |
| priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool, |
| GFP_DMA32 | GFP_ATOMIC, |
| &priv_ep->trb_pool_dma); |
| |
| if (!priv_ep->trb_pool) |
| return -ENOMEM; |
| |
| priv_ep->alloc_ring_size = ring_size; |
| } |
| |
| memset(priv_ep->trb_pool, 0, ring_size); |
| |
| priv_ep->num_trbs = num_trbs; |
| |
| if (!priv_ep->num) |
| return 0; |
| |
| /* Initialize the last TRB as Link TRB */ |
| link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1)); |
| |
| if (priv_ep->use_streams) { |
| /* |
| * For stream capable endpoints driver use single correct TRB. |
| * The last trb has zeroed cycle bit |
| */ |
| link_trb->control = 0; |
| } else { |
| link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma)); |
| link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE); |
| } |
| return 0; |
| } |
| |
| /** |
| * cdns3_ep_stall_flush - Stalls and flushes selected endpoint |
| * @priv_ep: endpoint object |
| * |
| * Endpoint must be selected before call to this function |
| */ |
| static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| int val; |
| |
| trace_cdns3_halt(priv_ep, 1, 1); |
| |
| writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL, |
| &priv_dev->regs->ep_cmd); |
| |
| /* wait for DFLUSH cleared */ |
| readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, |
| !(val & EP_CMD_DFLUSH), 1, 1000); |
| priv_ep->flags |= EP_STALLED; |
| priv_ep->flags &= ~EP_STALL_PENDING; |
| } |
| |
| /** |
| * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller. |
| * @priv_dev: extended gadget object |
| */ |
| void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev) |
| { |
| int i; |
| |
| writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf); |
| |
| cdns3_allow_enable_l1(priv_dev, 0); |
| priv_dev->hw_configured_flag = 0; |
| priv_dev->onchip_used_size = 0; |
| priv_dev->out_mem_is_allocated = 0; |
| priv_dev->wait_for_setup = 0; |
| priv_dev->using_streams = 0; |
| |
| for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) |
| if (priv_dev->eps[i]) |
| priv_dev->eps[i]->flags &= ~EP_CONFIGURED; |
| } |
| |
| /** |
| * cdns3_ep_inc_trb - increment a trb index. |
| * @index: Pointer to the TRB index to increment. |
| * @cs: Cycle state |
| * @trb_in_seg: number of TRBs in segment |
| * |
| * The index should never point to the link TRB. After incrementing, |
| * if it is point to the link TRB, wrap around to the beginning and revert |
| * cycle state bit The |
| * link TRB is always at the last TRB entry. |
| */ |
| static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg) |
| { |
| (*index)++; |
| if (*index == (trb_in_seg - 1)) { |
| *index = 0; |
| *cs ^= 1; |
| } |
| } |
| |
| /** |
| * cdns3_ep_inc_enq - increment endpoint's enqueue pointer |
| * @priv_ep: The endpoint whose enqueue pointer we're incrementing |
| */ |
| static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep) |
| { |
| priv_ep->free_trbs--; |
| cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs); |
| } |
| |
| /** |
| * cdns3_ep_inc_deq - increment endpoint's dequeue pointer |
| * @priv_ep: The endpoint whose dequeue pointer we're incrementing |
| */ |
| static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep) |
| { |
| priv_ep->free_trbs++; |
| cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs); |
| } |
| |
| /** |
| * cdns3_allow_enable_l1 - enable/disable permits to transition to L1. |
| * @priv_dev: Extended gadget object |
| * @enable: Enable/disable permit to transition to L1. |
| * |
| * If bit USB_CONF_L1EN is set and device receive Extended Token packet, |
| * then controller answer with ACK handshake. |
| * If bit USB_CONF_L1DS is set and device receive Extended Token packet, |
| * then controller answer with NYET handshake. |
| */ |
| void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable) |
| { |
| if (enable) |
| writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf); |
| else |
| writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf); |
| } |
| |
| enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev) |
| { |
| u32 reg; |
| |
| reg = readl(&priv_dev->regs->usb_sts); |
| |
| if (DEV_SUPERSPEED(reg)) |
| return USB_SPEED_SUPER; |
| else if (DEV_HIGHSPEED(reg)) |
| return USB_SPEED_HIGH; |
| else if (DEV_FULLSPEED(reg)) |
| return USB_SPEED_FULL; |
| else if (DEV_LOWSPEED(reg)) |
| return USB_SPEED_LOW; |
| return USB_SPEED_UNKNOWN; |
| } |
| |
| /** |
| * cdns3_start_all_request - add to ring all request not started |
| * @priv_dev: Extended gadget object |
| * @priv_ep: The endpoint for whom request will be started. |
| * |
| * Returns return ENOMEM if transfer ring i not enough TRBs to start |
| * all requests. |
| */ |
| static int cdns3_start_all_request(struct cdns3_device *priv_dev, |
| struct cdns3_endpoint *priv_ep) |
| { |
| struct usb_request *request; |
| int ret = 0; |
| u8 pending_empty = list_empty(&priv_ep->pending_req_list); |
| |
| /* |
| * If the last pending transfer is INTERNAL |
| * OR streams are enabled for this endpoint |
| * do NOT start new transfer till the last one is pending |
| */ |
| if (!pending_empty) { |
| struct cdns3_request *priv_req; |
| |
| request = cdns3_next_request(&priv_ep->pending_req_list); |
| priv_req = to_cdns3_request(request); |
| if ((priv_req->flags & REQUEST_INTERNAL) || |
| (priv_ep->flags & EP_TDLCHK_EN) || |
| priv_ep->use_streams) { |
| dev_dbg(priv_dev->dev, "Blocking external request\n"); |
| return ret; |
| } |
| } |
| |
| while (!list_empty(&priv_ep->deferred_req_list)) { |
| request = cdns3_next_request(&priv_ep->deferred_req_list); |
| |
| if (!priv_ep->use_streams) { |
| ret = cdns3_ep_run_transfer(priv_ep, request); |
| } else { |
| priv_ep->stream_sg_idx = 0; |
| ret = cdns3_ep_run_stream_transfer(priv_ep, request); |
| } |
| if (ret) |
| return ret; |
| |
| list_move_tail(&request->list, &priv_ep->pending_req_list); |
| if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN)) |
| break; |
| } |
| |
| priv_ep->flags &= ~EP_RING_FULL; |
| return ret; |
| } |
| |
| /* |
| * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set |
| * driver try to detect whether endpoint need additional internal |
| * buffer for unblocking on-chip FIFO buffer. This flag will be cleared |
| * if before first DESCMISS interrupt the DMA will be armed. |
| */ |
| #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \ |
| if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \ |
| priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \ |
| (reg) |= EP_STS_EN_DESCMISEN; \ |
| } } while (0) |
| |
| static void __cdns3_descmiss_copy_data(struct usb_request *request, |
| struct usb_request *descmiss_req) |
| { |
| int length = request->actual + descmiss_req->actual; |
| struct scatterlist *s = request->sg; |
| |
| if (!s) { |
| if (length <= request->length) { |
| memcpy(&((u8 *)request->buf)[request->actual], |
| descmiss_req->buf, |
| descmiss_req->actual); |
| request->actual = length; |
| } else { |
| /* It should never occures */ |
| request->status = -ENOMEM; |
| } |
| } else { |
| if (length <= sg_dma_len(s)) { |
| void *p = phys_to_virt(sg_dma_address(s)); |
| |
| memcpy(&((u8 *)p)[request->actual], |
| descmiss_req->buf, |
| descmiss_req->actual); |
| request->actual = length; |
| } else { |
| request->status = -ENOMEM; |
| } |
| } |
| } |
| |
| /** |
| * cdns3_wa2_descmiss_copy_data - copy data from internal requests to |
| * request queued by class driver. |
| * @priv_ep: extended endpoint object |
| * @request: request object |
| */ |
| static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep, |
| struct usb_request *request) |
| { |
| struct usb_request *descmiss_req; |
| struct cdns3_request *descmiss_priv_req; |
| |
| while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { |
| int chunk_end; |
| |
| descmiss_priv_req = |
| cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); |
| descmiss_req = &descmiss_priv_req->request; |
| |
| /* driver can't touch pending request */ |
| if (descmiss_priv_req->flags & REQUEST_PENDING) |
| break; |
| |
| chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH; |
| request->status = descmiss_req->status; |
| __cdns3_descmiss_copy_data(request, descmiss_req); |
| list_del_init(&descmiss_priv_req->list); |
| kfree(descmiss_req->buf); |
| cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req); |
| --priv_ep->wa2_counter; |
| |
| if (!chunk_end) |
| break; |
| } |
| } |
| |
| static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev, |
| struct cdns3_endpoint *priv_ep, |
| struct cdns3_request *priv_req) |
| { |
| if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN && |
| priv_req->flags & REQUEST_INTERNAL) { |
| struct usb_request *req; |
| |
| req = cdns3_next_request(&priv_ep->deferred_req_list); |
| |
| priv_ep->descmis_req = NULL; |
| |
| if (!req) |
| return NULL; |
| |
| /* unmap the gadget request before copying data */ |
| usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req, |
| priv_ep->dir); |
| |
| cdns3_wa2_descmiss_copy_data(priv_ep, req); |
| if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) && |
| req->length != req->actual) { |
| /* wait for next part of transfer */ |
| /* re-map the gadget request buffer*/ |
| usb_gadget_map_request_by_dev(priv_dev->sysdev, req, |
| usb_endpoint_dir_in(priv_ep->endpoint.desc)); |
| return NULL; |
| } |
| |
| if (req->status == -EINPROGRESS) |
| req->status = 0; |
| |
| list_del_init(&req->list); |
| cdns3_start_all_request(priv_dev, priv_ep); |
| return req; |
| } |
| |
| return &priv_req->request; |
| } |
| |
| static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev, |
| struct cdns3_endpoint *priv_ep, |
| struct cdns3_request *priv_req) |
| { |
| int deferred = 0; |
| |
| /* |
| * If transfer was queued before DESCMISS appear than we |
| * can disable handling of DESCMISS interrupt. Driver assumes that it |
| * can disable special treatment for this endpoint. |
| */ |
| if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { |
| u32 reg; |
| |
| cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir); |
| priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; |
| reg = readl(&priv_dev->regs->ep_sts_en); |
| reg &= ~EP_STS_EN_DESCMISEN; |
| trace_cdns3_wa2(priv_ep, "workaround disabled\n"); |
| writel(reg, &priv_dev->regs->ep_sts_en); |
| } |
| |
| if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { |
| u8 pending_empty = list_empty(&priv_ep->pending_req_list); |
| u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list); |
| |
| /* |
| * DESCMISS transfer has been finished, so data will be |
| * directly copied from internal allocated usb_request |
| * objects. |
| */ |
| if (pending_empty && !descmiss_empty && |
| !(priv_req->flags & REQUEST_INTERNAL)) { |
| cdns3_wa2_descmiss_copy_data(priv_ep, |
| &priv_req->request); |
| |
| trace_cdns3_wa2(priv_ep, "get internal stored data"); |
| |
| list_add_tail(&priv_req->request.list, |
| &priv_ep->pending_req_list); |
| cdns3_gadget_giveback(priv_ep, priv_req, |
| priv_req->request.status); |
| |
| /* |
| * Intentionally driver returns positive value as |
| * correct value. It informs that transfer has |
| * been finished. |
| */ |
| return EINPROGRESS; |
| } |
| |
| /* |
| * Driver will wait for completion DESCMISS transfer, |
| * before starts new, not DESCMISS transfer. |
| */ |
| if (!pending_empty && !descmiss_empty) { |
| trace_cdns3_wa2(priv_ep, "wait for pending transfer\n"); |
| deferred = 1; |
| } |
| |
| if (priv_req->flags & REQUEST_INTERNAL) |
| list_add_tail(&priv_req->list, |
| &priv_ep->wa2_descmiss_req_list); |
| } |
| |
| return deferred; |
| } |
| |
| static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_request *priv_req; |
| |
| while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { |
| u8 chain; |
| |
| priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); |
| chain = !!(priv_req->flags & REQUEST_INTERNAL_CH); |
| |
| trace_cdns3_wa2(priv_ep, "removes eldest request"); |
| |
| kfree(priv_req->request.buf); |
| cdns3_gadget_ep_free_request(&priv_ep->endpoint, |
| &priv_req->request); |
| list_del_init(&priv_req->list); |
| --priv_ep->wa2_counter; |
| |
| if (!chain) |
| break; |
| } |
| } |
| |
| /** |
| * cdns3_wa2_descmissing_packet - handles descriptor missing event. |
| * @priv_ep: extended gadget object |
| * |
| * This function is used only for WA2. For more information see Work around 2 |
| * description. |
| */ |
| static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_request *priv_req; |
| struct usb_request *request; |
| u8 pending_empty = list_empty(&priv_ep->pending_req_list); |
| |
| /* check for pending transfer */ |
| if (!pending_empty) { |
| trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n"); |
| return; |
| } |
| |
| if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { |
| priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; |
| priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN; |
| } |
| |
| trace_cdns3_wa2(priv_ep, "Description Missing detected\n"); |
| |
| if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) { |
| trace_cdns3_wa2(priv_ep, "WA2 overflow\n"); |
| cdns3_wa2_remove_old_request(priv_ep); |
| } |
| |
| request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint, |
| GFP_ATOMIC); |
| if (!request) |
| goto err; |
| |
| priv_req = to_cdns3_request(request); |
| priv_req->flags |= REQUEST_INTERNAL; |
| |
| /* if this field is still assigned it indicate that transfer related |
| * with this request has not been finished yet. Driver in this |
| * case simply allocate next request and assign flag REQUEST_INTERNAL_CH |
| * flag to previous one. It will indicate that current request is |
| * part of the previous one. |
| */ |
| if (priv_ep->descmis_req) |
| priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH; |
| |
| priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE, |
| GFP_ATOMIC); |
| priv_ep->wa2_counter++; |
| |
| if (!priv_req->request.buf) { |
| cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); |
| goto err; |
| } |
| |
| priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE; |
| priv_ep->descmis_req = priv_req; |
| |
| __cdns3_gadget_ep_queue(&priv_ep->endpoint, |
| &priv_ep->descmis_req->request, |
| GFP_ATOMIC); |
| |
| return; |
| |
| err: |
| dev_err(priv_ep->cdns3_dev->dev, |
| "Failed: No sufficient memory for DESCMIS\n"); |
| } |
| |
| static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev) |
| { |
| u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); |
| |
| if (tdl) { |
| u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl; |
| |
| writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL, |
| &priv_dev->regs->ep_cmd); |
| } |
| } |
| |
| static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev) |
| { |
| u32 ep_sts_reg; |
| |
| /* select EP0-out */ |
| cdns3_select_ep(priv_dev, 0); |
| |
| ep_sts_reg = readl(&priv_dev->regs->ep_sts); |
| |
| if (EP_STS_OUTQ_VAL(ep_sts_reg)) { |
| u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg); |
| struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num]; |
| |
| if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) && |
| outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) { |
| u8 pending_empty = list_empty(&outq_ep->pending_req_list); |
| |
| if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) || |
| (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) || |
| !pending_empty) { |
| } else { |
| u32 ep_sts_en_reg; |
| u32 ep_cmd_reg; |
| |
| cdns3_select_ep(priv_dev, outq_ep->num | |
| outq_ep->dir); |
| ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en); |
| ep_cmd_reg = readl(&priv_dev->regs->ep_cmd); |
| |
| outq_ep->flags |= EP_TDLCHK_EN; |
| cdns3_set_register_bit(&priv_dev->regs->ep_cfg, |
| EP_CFG_TDL_CHK); |
| |
| cdns3_wa2_enable_detection(priv_dev, outq_ep, |
| ep_sts_en_reg); |
| writel(ep_sts_en_reg, |
| &priv_dev->regs->ep_sts_en); |
| /* reset tdl value to zero */ |
| cdns3_wa2_reset_tdl(priv_dev); |
| /* |
| * Memory barrier - Reset tdl before ringing the |
| * doorbell. |
| */ |
| wmb(); |
| if (EP_CMD_DRDY & ep_cmd_reg) { |
| trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n"); |
| |
| } else { |
| trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n"); |
| /* |
| * ring doorbell to generate DESCMIS irq |
| */ |
| writel(EP_CMD_DRDY, |
| &priv_dev->regs->ep_cmd); |
| } |
| } |
| } |
| } |
| } |
| |
| /** |
| * cdns3_gadget_giveback - call struct usb_request's ->complete callback |
| * @priv_ep: The endpoint to whom the request belongs to |
| * @priv_req: The request we're giving back |
| * @status: completion code for the request |
| * |
| * Must be called with controller's lock held and interrupts disabled. This |
| * function will unmap @req and call its ->complete() callback to notify upper |
| * layers that it has completed. |
| */ |
| void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep, |
| struct cdns3_request *priv_req, |
| int status) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| struct usb_request *request = &priv_req->request; |
| |
| list_del_init(&request->list); |
| |
| if (request->status == -EINPROGRESS) |
| request->status = status; |
| |
| usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request, |
| priv_ep->dir); |
| |
| if ((priv_req->flags & REQUEST_UNALIGNED) && |
| priv_ep->dir == USB_DIR_OUT && !request->status) { |
| /* Make DMA buffer CPU accessible */ |
| dma_sync_single_for_cpu(priv_dev->sysdev, |
| priv_req->aligned_buf->dma, |
| priv_req->aligned_buf->size, |
| priv_req->aligned_buf->dir); |
| memcpy(request->buf, priv_req->aligned_buf->buf, |
| request->length); |
| } |
| |
| priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED); |
| /* All TRBs have finished, clear the counter */ |
| priv_req->finished_trb = 0; |
| trace_cdns3_gadget_giveback(priv_req); |
| |
| if (priv_dev->dev_ver < DEV_VER_V2) { |
| request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep, |
| priv_req); |
| if (!request) |
| return; |
| } |
| |
| if (request->complete) { |
| spin_unlock(&priv_dev->lock); |
| usb_gadget_giveback_request(&priv_ep->endpoint, |
| request); |
| spin_lock(&priv_dev->lock); |
| } |
| |
| if (request->buf == priv_dev->zlp_buf) |
| cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); |
| } |
| |
| static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep) |
| { |
| /* Work around for stale data address in TRB*/ |
| if (priv_ep->wa1_set) { |
| trace_cdns3_wa1(priv_ep, "restore cycle bit"); |
| |
| priv_ep->wa1_set = 0; |
| priv_ep->wa1_trb_index = 0xFFFF; |
| if (priv_ep->wa1_cycle_bit) { |
| priv_ep->wa1_trb->control = |
| priv_ep->wa1_trb->control | cpu_to_le32(0x1); |
| } else { |
| priv_ep->wa1_trb->control = |
| priv_ep->wa1_trb->control & cpu_to_le32(~0x1); |
| } |
| } |
| } |
| |
| static void cdns3_free_aligned_request_buf(struct work_struct *work) |
| { |
| struct cdns3_device *priv_dev = container_of(work, struct cdns3_device, |
| aligned_buf_wq); |
| struct cdns3_aligned_buf *buf, *tmp; |
| unsigned long flags; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| |
| list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) { |
| if (!buf->in_use) { |
| list_del(&buf->list); |
| |
| /* |
| * Re-enable interrupts to free DMA capable memory. |
| * Driver can't free this memory with disabled |
| * interrupts. |
| */ |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| dma_free_noncoherent(priv_dev->sysdev, buf->size, |
| buf->buf, buf->dma, buf->dir); |
| kfree(buf); |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| } |
| } |
| |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| } |
| |
| static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req) |
| { |
| struct cdns3_endpoint *priv_ep = priv_req->priv_ep; |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| struct cdns3_aligned_buf *buf; |
| |
| /* check if buffer is aligned to 8. */ |
| if (!((uintptr_t)priv_req->request.buf & 0x7)) |
| return 0; |
| |
| buf = priv_req->aligned_buf; |
| |
| if (!buf || priv_req->request.length > buf->size) { |
| buf = kzalloc(sizeof(*buf), GFP_ATOMIC); |
| if (!buf) |
| return -ENOMEM; |
| |
| buf->size = priv_req->request.length; |
| buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ? |
| DMA_TO_DEVICE : DMA_FROM_DEVICE; |
| |
| buf->buf = dma_alloc_noncoherent(priv_dev->sysdev, |
| buf->size, |
| &buf->dma, |
| buf->dir, |
| GFP_ATOMIC); |
| if (!buf->buf) { |
| kfree(buf); |
| return -ENOMEM; |
| } |
| |
| if (priv_req->aligned_buf) { |
| trace_cdns3_free_aligned_request(priv_req); |
| priv_req->aligned_buf->in_use = 0; |
| queue_work(system_freezable_wq, |
| &priv_dev->aligned_buf_wq); |
| } |
| |
| buf->in_use = 1; |
| priv_req->aligned_buf = buf; |
| |
| list_add_tail(&buf->list, |
| &priv_dev->aligned_buf_list); |
| } |
| |
| if (priv_ep->dir == USB_DIR_IN) { |
| /* Make DMA buffer CPU accessible */ |
| dma_sync_single_for_cpu(priv_dev->sysdev, |
| buf->dma, buf->size, buf->dir); |
| memcpy(buf->buf, priv_req->request.buf, |
| priv_req->request.length); |
| } |
| |
| /* Transfer DMA buffer ownership back to device */ |
| dma_sync_single_for_device(priv_dev->sysdev, |
| buf->dma, buf->size, buf->dir); |
| |
| priv_req->flags |= REQUEST_UNALIGNED; |
| trace_cdns3_prepare_aligned_request(priv_req); |
| |
| return 0; |
| } |
| |
| static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep, |
| struct cdns3_trb *trb) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| |
| if (!priv_ep->wa1_set) { |
| u32 doorbell; |
| |
| doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); |
| |
| if (doorbell) { |
| priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0; |
| priv_ep->wa1_set = 1; |
| priv_ep->wa1_trb = trb; |
| priv_ep->wa1_trb_index = priv_ep->enqueue; |
| trace_cdns3_wa1(priv_ep, "set guard"); |
| return 0; |
| } |
| } |
| return 1; |
| } |
| |
| static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev, |
| struct cdns3_endpoint *priv_ep) |
| { |
| int dma_index; |
| u32 doorbell; |
| |
| doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); |
| dma_index = cdns3_get_dma_pos(priv_dev, priv_ep); |
| |
| if (!doorbell || dma_index != priv_ep->wa1_trb_index) |
| cdns3_wa1_restore_cycle_bit(priv_ep); |
| } |
| |
| static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep, |
| struct usb_request *request) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| struct cdns3_request *priv_req; |
| struct cdns3_trb *trb; |
| dma_addr_t trb_dma; |
| int address; |
| u32 control; |
| u32 length; |
| u32 tdl; |
| unsigned int sg_idx = priv_ep->stream_sg_idx; |
| |
| priv_req = to_cdns3_request(request); |
| address = priv_ep->endpoint.desc->bEndpointAddress; |
| |
| priv_ep->flags |= EP_PENDING_REQUEST; |
| |
| /* must allocate buffer aligned to 8 */ |
| if (priv_req->flags & REQUEST_UNALIGNED) |
| trb_dma = priv_req->aligned_buf->dma; |
| else |
| trb_dma = request->dma; |
| |
| /* For stream capable endpoints driver use only single TD. */ |
| trb = priv_ep->trb_pool + priv_ep->enqueue; |
| priv_req->start_trb = priv_ep->enqueue; |
| priv_req->end_trb = priv_req->start_trb; |
| priv_req->trb = trb; |
| |
| cdns3_select_ep(priv_ep->cdns3_dev, address); |
| |
| control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE | |
| TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP; |
| |
| if (!request->num_sgs) { |
| trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); |
| length = request->length; |
| } else { |
| trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address)); |
| length = request->sg[sg_idx].length; |
| } |
| |
| tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket); |
| |
| trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length)); |
| |
| /* |
| * For DEV_VER_V2 controller version we have enabled |
| * USB_CONF2_EN_TDL_TRB in DMULT configuration. |
| * This enables TDL calculation based on TRB, hence setting TDL in TRB. |
| */ |
| if (priv_dev->dev_ver >= DEV_VER_V2) { |
| if (priv_dev->gadget.speed == USB_SPEED_SUPER) |
| trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl)); |
| } |
| priv_req->flags |= REQUEST_PENDING; |
| |
| trb->control = cpu_to_le32(control); |
| |
| trace_cdns3_prepare_trb(priv_ep, priv_req->trb); |
| |
| /* |
| * Memory barrier - Cycle Bit must be set before trb->length and |
| * trb->buffer fields. |
| */ |
| wmb(); |
| |
| /* always first element */ |
| writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma), |
| &priv_dev->regs->ep_traddr); |
| |
| if (!(priv_ep->flags & EP_STALLED)) { |
| trace_cdns3_ring(priv_ep); |
| /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/ |
| writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts); |
| |
| priv_ep->prime_flag = false; |
| |
| /* |
| * Controller version DEV_VER_V2 tdl calculation |
| * is based on TRB |
| */ |
| |
| if (priv_dev->dev_ver < DEV_VER_V2) |
| writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, |
| &priv_dev->regs->ep_cmd); |
| else if (priv_dev->dev_ver > DEV_VER_V2) |
| writel(tdl, &priv_dev->regs->ep_tdl); |
| |
| priv_ep->last_stream_id = priv_req->request.stream_id; |
| writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); |
| writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) | |
| EP_CMD_ERDY, &priv_dev->regs->ep_cmd); |
| |
| trace_cdns3_doorbell_epx(priv_ep->name, |
| readl(&priv_dev->regs->ep_traddr)); |
| } |
| |
| /* WORKAROUND for transition to L0 */ |
| __cdns3_gadget_wakeup(priv_dev); |
| |
| return 0; |
| } |
| |
| static void cdns3_rearm_drdy_if_needed(struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| |
| if (priv_dev->dev_ver < DEV_VER_V3) |
| return; |
| |
| if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) { |
| writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts); |
| writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); |
| } |
| } |
| |
| /** |
| * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware |
| * @priv_ep: endpoint object |
| * @request: request object |
| * |
| * Returns zero on success or negative value on failure |
| */ |
| static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, |
| struct usb_request *request) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| struct cdns3_request *priv_req; |
| struct cdns3_trb *trb; |
| struct cdns3_trb *link_trb = NULL; |
| dma_addr_t trb_dma; |
| u32 togle_pcs = 1; |
| int sg_iter = 0; |
| int num_trb; |
| int address; |
| u32 control; |
| int pcs; |
| u16 total_tdl = 0; |
| struct scatterlist *s = NULL; |
| bool sg_supported = !!(request->num_mapped_sgs); |
| |
| if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) |
| num_trb = priv_ep->interval; |
| else |
| num_trb = sg_supported ? request->num_mapped_sgs : 1; |
| |
| if (num_trb > priv_ep->free_trbs) { |
| priv_ep->flags |= EP_RING_FULL; |
| return -ENOBUFS; |
| } |
| |
| priv_req = to_cdns3_request(request); |
| address = priv_ep->endpoint.desc->bEndpointAddress; |
| |
| priv_ep->flags |= EP_PENDING_REQUEST; |
| |
| /* must allocate buffer aligned to 8 */ |
| if (priv_req->flags & REQUEST_UNALIGNED) |
| trb_dma = priv_req->aligned_buf->dma; |
| else |
| trb_dma = request->dma; |
| |
| trb = priv_ep->trb_pool + priv_ep->enqueue; |
| priv_req->start_trb = priv_ep->enqueue; |
| priv_req->trb = trb; |
| |
| cdns3_select_ep(priv_ep->cdns3_dev, address); |
| |
| /* prepare ring */ |
| if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) { |
| int doorbell, dma_index; |
| u32 ch_bit = 0; |
| |
| doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); |
| dma_index = cdns3_get_dma_pos(priv_dev, priv_ep); |
| |
| /* Driver can't update LINK TRB if it is current processed. */ |
| if (doorbell && dma_index == priv_ep->num_trbs - 1) { |
| priv_ep->flags |= EP_DEFERRED_DRDY; |
| return -ENOBUFS; |
| } |
| |
| /*updating C bt in Link TRB before starting DMA*/ |
| link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1); |
| /* |
| * For TRs size equal 2 enabling TRB_CHAIN for epXin causes |
| * that DMA stuck at the LINK TRB. |
| * On the other hand, removing TRB_CHAIN for longer TRs for |
| * epXout cause that DMA stuck after handling LINK TRB. |
| * To eliminate this strange behavioral driver set TRB_CHAIN |
| * bit only for TR size > 2. |
| */ |
| if (priv_ep->type == USB_ENDPOINT_XFER_ISOC || |
| TRBS_PER_SEGMENT > 2) |
| ch_bit = TRB_CHAIN; |
| |
| link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) | |
| TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit); |
| } |
| |
| if (priv_dev->dev_ver <= DEV_VER_V2) |
| togle_pcs = cdns3_wa1_update_guard(priv_ep, trb); |
| |
| if (sg_supported) |
| s = request->sg; |
| |
| /* set incorrect Cycle Bit for first trb*/ |
| control = priv_ep->pcs ? 0 : TRB_CYCLE; |
| trb->length = 0; |
| if (priv_dev->dev_ver >= DEV_VER_V2) { |
| u16 td_size; |
| |
| td_size = DIV_ROUND_UP(request->length, |
| priv_ep->endpoint.maxpacket); |
| if (priv_dev->gadget.speed == USB_SPEED_SUPER) |
| trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size)); |
| else |
| control |= TRB_TDL_HS_SIZE(td_size); |
| } |
| |
| do { |
| u32 length; |
| |
| /* fill TRB */ |
| control |= TRB_TYPE(TRB_NORMAL); |
| if (sg_supported) { |
| trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s))); |
| length = sg_dma_len(s); |
| } else { |
| trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); |
| length = request->length; |
| } |
| |
| if (priv_ep->flags & EP_TDLCHK_EN) |
| total_tdl += DIV_ROUND_UP(length, |
| priv_ep->endpoint.maxpacket); |
| |
| trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) | |
| TRB_LEN(length)); |
| pcs = priv_ep->pcs ? TRB_CYCLE : 0; |
| |
| /* |
| * first trb should be prepared as last to avoid processing |
| * transfer to early |
| */ |
| if (sg_iter != 0) |
| control |= pcs; |
| |
| if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) { |
| control |= TRB_IOC | TRB_ISP; |
| } else { |
| /* for last element in TD or in SG list */ |
| if (sg_iter == (num_trb - 1) && sg_iter != 0) |
| control |= pcs | TRB_IOC | TRB_ISP; |
| } |
| |
| if (sg_iter) |
| trb->control = cpu_to_le32(control); |
| else |
| priv_req->trb->control = cpu_to_le32(control); |
| |
| if (sg_supported) { |
| trb->control |= cpu_to_le32(TRB_ISP); |
| /* Don't set chain bit for last TRB */ |
| if (sg_iter < num_trb - 1) |
| trb->control |= cpu_to_le32(TRB_CHAIN); |
| |
| s = sg_next(s); |
| } |
| |
| control = 0; |
| ++sg_iter; |
| priv_req->end_trb = priv_ep->enqueue; |
| cdns3_ep_inc_enq(priv_ep); |
| trb = priv_ep->trb_pool + priv_ep->enqueue; |
| trb->length = 0; |
| } while (sg_iter < num_trb); |
| |
| trb = priv_req->trb; |
| |
| priv_req->flags |= REQUEST_PENDING; |
| priv_req->num_of_trb = num_trb; |
| |
| if (sg_iter == 1) |
| trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP); |
| |
| if (priv_dev->dev_ver < DEV_VER_V2 && |
| (priv_ep->flags & EP_TDLCHK_EN)) { |
| u16 tdl = total_tdl; |
| u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); |
| |
| if (tdl > EP_CMD_TDL_MAX) { |
| tdl = EP_CMD_TDL_MAX; |
| priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX; |
| } |
| |
| if (old_tdl < tdl) { |
| tdl -= old_tdl; |
| writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, |
| &priv_dev->regs->ep_cmd); |
| } |
| } |
| |
| /* |
| * Memory barrier - cycle bit must be set before other filds in trb. |
| */ |
| wmb(); |
| |
| /* give the TD to the consumer*/ |
| if (togle_pcs) |
| trb->control = trb->control ^ cpu_to_le32(1); |
| |
| if (priv_dev->dev_ver <= DEV_VER_V2) |
| cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep); |
| |
| if (num_trb > 1) { |
| int i = 0; |
| |
| while (i < num_trb) { |
| trace_cdns3_prepare_trb(priv_ep, trb + i); |
| if (trb + i == link_trb) { |
| trb = priv_ep->trb_pool; |
| num_trb = num_trb - i; |
| i = 0; |
| } else { |
| i++; |
| } |
| } |
| } else { |
| trace_cdns3_prepare_trb(priv_ep, priv_req->trb); |
| } |
| |
| /* |
| * Memory barrier - Cycle Bit must be set before trb->length and |
| * trb->buffer fields. |
| */ |
| wmb(); |
| |
| /* |
| * For DMULT mode we can set address to transfer ring only once after |
| * enabling endpoint. |
| */ |
| if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) { |
| /* |
| * Until SW is not ready to handle the OUT transfer the ISO OUT |
| * Endpoint should be disabled (EP_CFG.ENABLE = 0). |
| * EP_CFG_ENABLE must be set before updating ep_traddr. |
| */ |
| if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir && |
| !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) { |
| priv_ep->flags |= EP_QUIRK_ISO_OUT_EN; |
| cdns3_set_register_bit(&priv_dev->regs->ep_cfg, |
| EP_CFG_ENABLE); |
| } |
| |
| writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma + |
| priv_req->start_trb * TRB_SIZE), |
| &priv_dev->regs->ep_traddr); |
| |
| priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR; |
| } |
| |
| if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) { |
| trace_cdns3_ring(priv_ep); |
| /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/ |
| writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts); |
| writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); |
| cdns3_rearm_drdy_if_needed(priv_ep); |
| trace_cdns3_doorbell_epx(priv_ep->name, |
| readl(&priv_dev->regs->ep_traddr)); |
| } |
| |
| /* WORKAROUND for transition to L0 */ |
| __cdns3_gadget_wakeup(priv_dev); |
| |
| return 0; |
| } |
| |
| void cdns3_set_hw_configuration(struct cdns3_device *priv_dev) |
| { |
| struct cdns3_endpoint *priv_ep; |
| struct usb_ep *ep; |
| |
| if (priv_dev->hw_configured_flag) |
| return; |
| |
| writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf); |
| |
| cdns3_set_register_bit(&priv_dev->regs->usb_conf, |
| USB_CONF_U1EN | USB_CONF_U2EN); |
| |
| priv_dev->hw_configured_flag = 1; |
| |
| list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { |
| if (ep->enabled) { |
| priv_ep = ep_to_cdns3_ep(ep); |
| cdns3_start_all_request(priv_dev, priv_ep); |
| } |
| } |
| |
| cdns3_allow_enable_l1(priv_dev, 1); |
| } |
| |
| /** |
| * cdns3_trb_handled - check whether trb has been handled by DMA |
| * |
| * @priv_ep: extended endpoint object. |
| * @priv_req: request object for checking |
| * |
| * Endpoint must be selected before invoking this function. |
| * |
| * Returns false if request has not been handled by DMA, else returns true. |
| * |
| * SR - start ring |
| * ER - end ring |
| * DQ = priv_ep->dequeue - dequeue position |
| * EQ = priv_ep->enqueue - enqueue position |
| * ST = priv_req->start_trb - index of first TRB in transfer ring |
| * ET = priv_req->end_trb - index of last TRB in transfer ring |
| * CI = current_index - index of processed TRB by DMA. |
| * |
| * As first step, we check if the TRB between the ST and ET. |
| * Then, we check if cycle bit for index priv_ep->dequeue |
| * is correct. |
| * |
| * some rules: |
| * 1. priv_ep->dequeue never equals to current_index. |
| * 2 priv_ep->enqueue never exceed priv_ep->dequeue |
| * 3. exception: priv_ep->enqueue == priv_ep->dequeue |
| * and priv_ep->free_trbs is zero. |
| * This case indicate that TR is full. |
| * |
| * At below two cases, the request have been handled. |
| * Case 1 - priv_ep->dequeue < current_index |
| * SR ... EQ ... DQ ... CI ... ER |
| * SR ... DQ ... CI ... EQ ... ER |
| * |
| * Case 2 - priv_ep->dequeue > current_index |
| * This situation takes place when CI go through the LINK TRB at the end of |
| * transfer ring. |
| * SR ... CI ... EQ ... DQ ... ER |
| */ |
| static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep, |
| struct cdns3_request *priv_req) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| struct cdns3_trb *trb; |
| int current_index = 0; |
| int handled = 0; |
| int doorbell; |
| |
| current_index = cdns3_get_dma_pos(priv_dev, priv_ep); |
| doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); |
| |
| /* current trb doesn't belong to this request */ |
| if (priv_req->start_trb < priv_req->end_trb) { |
| if (priv_ep->dequeue > priv_req->end_trb) |
| goto finish; |
| |
| if (priv_ep->dequeue < priv_req->start_trb) |
| goto finish; |
| } |
| |
| if ((priv_req->start_trb > priv_req->end_trb) && |
| (priv_ep->dequeue > priv_req->end_trb) && |
| (priv_ep->dequeue < priv_req->start_trb)) |
| goto finish; |
| |
| if ((priv_req->start_trb == priv_req->end_trb) && |
| (priv_ep->dequeue != priv_req->end_trb)) |
| goto finish; |
| |
| trb = &priv_ep->trb_pool[priv_ep->dequeue]; |
| |
| if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs) |
| goto finish; |
| |
| if (doorbell == 1 && current_index == priv_ep->dequeue) |
| goto finish; |
| |
| /* The corner case for TRBS_PER_SEGMENT equal 2). */ |
| if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { |
| handled = 1; |
| goto finish; |
| } |
| |
| if (priv_ep->enqueue == priv_ep->dequeue && |
| priv_ep->free_trbs == 0) { |
| handled = 1; |
| } else if (priv_ep->dequeue < current_index) { |
| if ((current_index == (priv_ep->num_trbs - 1)) && |
| !priv_ep->dequeue) |
| goto finish; |
| |
| handled = 1; |
| } else if (priv_ep->dequeue > current_index) { |
| handled = 1; |
| } |
| |
| finish: |
| trace_cdns3_request_handled(priv_req, current_index, handled); |
| |
| return handled; |
| } |
| |
| static void cdns3_transfer_completed(struct cdns3_device *priv_dev, |
| struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_request *priv_req; |
| struct usb_request *request; |
| struct cdns3_trb *trb; |
| bool request_handled = false; |
| bool transfer_end = false; |
| |
| while (!list_empty(&priv_ep->pending_req_list)) { |
| request = cdns3_next_request(&priv_ep->pending_req_list); |
| priv_req = to_cdns3_request(request); |
| |
| trb = priv_ep->trb_pool + priv_ep->dequeue; |
| |
| /* The TRB was changed as link TRB, and the request was handled at ep_dequeue */ |
| while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) { |
| trace_cdns3_complete_trb(priv_ep, trb); |
| cdns3_ep_inc_deq(priv_ep); |
| trb = priv_ep->trb_pool + priv_ep->dequeue; |
| } |
| |
| if (!request->stream_id) { |
| /* Re-select endpoint. It could be changed by other CPU |
| * during handling usb_gadget_giveback_request. |
| */ |
| cdns3_select_ep(priv_dev, priv_ep->endpoint.address); |
| |
| while (cdns3_trb_handled(priv_ep, priv_req)) { |
| priv_req->finished_trb++; |
| if (priv_req->finished_trb >= priv_req->num_of_trb) |
| request_handled = true; |
| |
| trb = priv_ep->trb_pool + priv_ep->dequeue; |
| trace_cdns3_complete_trb(priv_ep, trb); |
| |
| if (!transfer_end) |
| request->actual += |
| TRB_LEN(le32_to_cpu(trb->length)); |
| |
| if (priv_req->num_of_trb > 1 && |
| le32_to_cpu(trb->control) & TRB_SMM) |
| transfer_end = true; |
| |
| cdns3_ep_inc_deq(priv_ep); |
| } |
| |
| if (request_handled) { |
| cdns3_gadget_giveback(priv_ep, priv_req, 0); |
| request_handled = false; |
| transfer_end = false; |
| } else { |
| goto prepare_next_td; |
| } |
| |
| if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && |
| TRBS_PER_SEGMENT == 2) |
| break; |
| } else { |
| /* Re-select endpoint. It could be changed by other CPU |
| * during handling usb_gadget_giveback_request. |
| */ |
| cdns3_select_ep(priv_dev, priv_ep->endpoint.address); |
| |
| trb = priv_ep->trb_pool; |
| trace_cdns3_complete_trb(priv_ep, trb); |
| |
| if (trb != priv_req->trb) |
| dev_warn(priv_dev->dev, |
| "request_trb=0x%p, queue_trb=0x%p\n", |
| priv_req->trb, trb); |
| |
| request->actual += TRB_LEN(le32_to_cpu(trb->length)); |
| |
| if (!request->num_sgs || |
| (request->num_sgs == (priv_ep->stream_sg_idx + 1))) { |
| priv_ep->stream_sg_idx = 0; |
| cdns3_gadget_giveback(priv_ep, priv_req, 0); |
| } else { |
| priv_ep->stream_sg_idx++; |
| cdns3_ep_run_stream_transfer(priv_ep, request); |
| } |
| break; |
| } |
| } |
| priv_ep->flags &= ~EP_PENDING_REQUEST; |
| |
| prepare_next_td: |
| if (!(priv_ep->flags & EP_STALLED) && |
| !(priv_ep->flags & EP_STALL_PENDING)) |
| cdns3_start_all_request(priv_dev, priv_ep); |
| } |
| |
| void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| |
| cdns3_wa1_restore_cycle_bit(priv_ep); |
| |
| if (rearm) { |
| trace_cdns3_ring(priv_ep); |
| |
| /* Cycle Bit must be updated before arming DMA. */ |
| wmb(); |
| writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); |
| |
| __cdns3_gadget_wakeup(priv_dev); |
| |
| trace_cdns3_doorbell_epx(priv_ep->name, |
| readl(&priv_dev->regs->ep_traddr)); |
| } |
| } |
| |
| static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep) |
| { |
| u16 tdl = priv_ep->pending_tdl; |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| |
| if (tdl > EP_CMD_TDL_MAX) { |
| tdl = EP_CMD_TDL_MAX; |
| priv_ep->pending_tdl -= EP_CMD_TDL_MAX; |
| } else { |
| priv_ep->pending_tdl = 0; |
| } |
| |
| writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd); |
| } |
| |
| /** |
| * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint |
| * @priv_ep: endpoint object |
| * |
| * Returns 0 |
| */ |
| static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| u32 ep_sts_reg; |
| struct usb_request *deferred_request; |
| struct usb_request *pending_request; |
| u32 tdl = 0; |
| |
| cdns3_select_ep(priv_dev, priv_ep->endpoint.address); |
| |
| trace_cdns3_epx_irq(priv_dev, priv_ep); |
| |
| ep_sts_reg = readl(&priv_dev->regs->ep_sts); |
| writel(ep_sts_reg, &priv_dev->regs->ep_sts); |
| |
| if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) { |
| bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY); |
| |
| tdl = cdns3_get_tdl(priv_dev); |
| |
| /* |
| * Continue the previous transfer: |
| * There is some racing between ERDY and PRIME. The device send |
| * ERDY and almost in the same time Host send PRIME. It cause |
| * that host ignore the ERDY packet and driver has to send it |
| * again. |
| */ |
| if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) || |
| EP_STS_HOSTPP(ep_sts_reg))) { |
| writel(EP_CMD_ERDY | |
| EP_CMD_ERDY_SID(priv_ep->last_stream_id), |
| &priv_dev->regs->ep_cmd); |
| ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC); |
| } else { |
| priv_ep->prime_flag = true; |
| |
| pending_request = cdns3_next_request(&priv_ep->pending_req_list); |
| deferred_request = cdns3_next_request(&priv_ep->deferred_req_list); |
| |
| if (deferred_request && !pending_request) { |
| cdns3_start_all_request(priv_dev, priv_ep); |
| } |
| } |
| } |
| |
| if (ep_sts_reg & EP_STS_TRBERR) { |
| if (priv_ep->flags & EP_STALL_PENDING && |
| !(ep_sts_reg & EP_STS_DESCMIS && |
| priv_dev->dev_ver < DEV_VER_V2)) { |
| cdns3_ep_stall_flush(priv_ep); |
| } |
| |
| /* |
| * For isochronous transfer driver completes request on |
| * IOC or on TRBERR. IOC appears only when device receive |
| * OUT data packet. If host disable stream or lost some packet |
| * then the only way to finish all queued transfer is to do it |
| * on TRBERR event. |
| */ |
| if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && |
| !priv_ep->wa1_set) { |
| if (!priv_ep->dir) { |
| u32 ep_cfg = readl(&priv_dev->regs->ep_cfg); |
| |
| ep_cfg &= ~EP_CFG_ENABLE; |
| writel(ep_cfg, &priv_dev->regs->ep_cfg); |
| priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN; |
| } |
| cdns3_transfer_completed(priv_dev, priv_ep); |
| } else if (!(priv_ep->flags & EP_STALLED) && |
| !(priv_ep->flags & EP_STALL_PENDING)) { |
| if (priv_ep->flags & EP_DEFERRED_DRDY) { |
| priv_ep->flags &= ~EP_DEFERRED_DRDY; |
| cdns3_start_all_request(priv_dev, priv_ep); |
| } else { |
| cdns3_rearm_transfer(priv_ep, |
| priv_ep->wa1_set); |
| } |
| } |
| } |
| |
| if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) || |
| (ep_sts_reg & EP_STS_IOT)) { |
| if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { |
| if (ep_sts_reg & EP_STS_ISP) |
| priv_ep->flags |= EP_QUIRK_END_TRANSFER; |
| else |
| priv_ep->flags &= ~EP_QUIRK_END_TRANSFER; |
| } |
| |
| if (!priv_ep->use_streams) { |
| if ((ep_sts_reg & EP_STS_IOC) || |
| (ep_sts_reg & EP_STS_ISP)) { |
| cdns3_transfer_completed(priv_dev, priv_ep); |
| } else if ((priv_ep->flags & EP_TDLCHK_EN) & |
| priv_ep->pending_tdl) { |
| /* handle IOT with pending tdl */ |
| cdns3_reprogram_tdl(priv_ep); |
| } |
| } else if (priv_ep->dir == USB_DIR_OUT) { |
| priv_ep->ep_sts_pending |= ep_sts_reg; |
| } else if (ep_sts_reg & EP_STS_IOT) { |
| cdns3_transfer_completed(priv_dev, priv_ep); |
| } |
| } |
| |
| /* |
| * MD_EXIT interrupt sets when stream capable endpoint exits |
| * from MOVE DATA state of Bulk IN/OUT stream protocol state machine |
| */ |
| if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) && |
| (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) { |
| priv_ep->ep_sts_pending = 0; |
| cdns3_transfer_completed(priv_dev, priv_ep); |
| } |
| |
| /* |
| * WA2: this condition should only be meet when |
| * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or |
| * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN. |
| * In other cases this interrupt will be disabled. |
| */ |
| if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 && |
| !(priv_ep->flags & EP_STALLED)) |
| cdns3_wa2_descmissing_packet(priv_ep); |
| |
| return 0; |
| } |
| |
| static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev) |
| { |
| if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) |
| priv_dev->gadget_driver->disconnect(&priv_dev->gadget); |
| } |
| |
| /** |
| * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device |
| * @priv_dev: extended gadget object |
| * @usb_ists: bitmap representation of device's reported interrupts |
| * (usb_ists register value) |
| */ |
| static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev, |
| u32 usb_ists) |
| __must_hold(&priv_dev->lock) |
| { |
| int speed = 0; |
| |
| trace_cdns3_usb_irq(priv_dev, usb_ists); |
| if (usb_ists & USB_ISTS_L1ENTI) { |
| /* |
| * WORKAROUND: CDNS3 controller has issue with hardware resuming |
| * from L1. To fix it, if any DMA transfer is pending driver |
| * must starts driving resume signal immediately. |
| */ |
| if (readl(&priv_dev->regs->drbl)) |
| __cdns3_gadget_wakeup(priv_dev); |
| } |
| |
| /* Connection detected */ |
| if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) { |
| speed = cdns3_get_speed(priv_dev); |
| priv_dev->gadget.speed = speed; |
| usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED); |
| cdns3_ep0_config(priv_dev); |
| } |
| |
| /* Disconnection detected */ |
| if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) { |
| spin_unlock(&priv_dev->lock); |
| cdns3_disconnect_gadget(priv_dev); |
| spin_lock(&priv_dev->lock); |
| priv_dev->gadget.speed = USB_SPEED_UNKNOWN; |
| usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED); |
| cdns3_hw_reset_eps_config(priv_dev); |
| } |
| |
| if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) { |
| if (priv_dev->gadget_driver && |
| priv_dev->gadget_driver->suspend) { |
| spin_unlock(&priv_dev->lock); |
| priv_dev->gadget_driver->suspend(&priv_dev->gadget); |
| spin_lock(&priv_dev->lock); |
| } |
| } |
| |
| if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) { |
| if (priv_dev->gadget_driver && |
| priv_dev->gadget_driver->resume) { |
| spin_unlock(&priv_dev->lock); |
| priv_dev->gadget_driver->resume(&priv_dev->gadget); |
| spin_lock(&priv_dev->lock); |
| } |
| } |
| |
| /* reset*/ |
| if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) { |
| if (priv_dev->gadget_driver) { |
| spin_unlock(&priv_dev->lock); |
| usb_gadget_udc_reset(&priv_dev->gadget, |
| priv_dev->gadget_driver); |
| spin_lock(&priv_dev->lock); |
| |
| /*read again to check the actual speed*/ |
| speed = cdns3_get_speed(priv_dev); |
| priv_dev->gadget.speed = speed; |
| cdns3_hw_reset_eps_config(priv_dev); |
| cdns3_ep0_config(priv_dev); |
| } |
| } |
| } |
| |
| /** |
| * cdns3_device_irq_handler - interrupt handler for device part of controller |
| * |
| * @irq: irq number for cdns3 core device |
| * @data: structure of cdns3 |
| * |
| * Returns IRQ_HANDLED or IRQ_NONE |
| */ |
| static irqreturn_t cdns3_device_irq_handler(int irq, void *data) |
| { |
| struct cdns3_device *priv_dev = data; |
| struct cdns *cdns = dev_get_drvdata(priv_dev->dev); |
| irqreturn_t ret = IRQ_NONE; |
| u32 reg; |
| |
| if (cdns->in_lpm) |
| return ret; |
| |
| /* check USB device interrupt */ |
| reg = readl(&priv_dev->regs->usb_ists); |
| if (reg) { |
| /* After masking interrupts the new interrupts won't be |
| * reported in usb_ists/ep_ists. In order to not lose some |
| * of them driver disables only detected interrupts. |
| * They will be enabled ASAP after clearing source of |
| * interrupt. This an unusual behavior only applies to |
| * usb_ists register. |
| */ |
| reg = ~reg & readl(&priv_dev->regs->usb_ien); |
| /* mask deferred interrupt. */ |
| writel(reg, &priv_dev->regs->usb_ien); |
| ret = IRQ_WAKE_THREAD; |
| } |
| |
| /* check endpoint interrupt */ |
| reg = readl(&priv_dev->regs->ep_ists); |
| if (reg) { |
| writel(0, &priv_dev->regs->ep_ien); |
| ret = IRQ_WAKE_THREAD; |
| } |
| |
| return ret; |
| } |
| |
| /** |
| * cdns3_device_thread_irq_handler - interrupt handler for device part |
| * of controller |
| * |
| * @irq: irq number for cdns3 core device |
| * @data: structure of cdns3 |
| * |
| * Returns IRQ_HANDLED or IRQ_NONE |
| */ |
| static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data) |
| { |
| struct cdns3_device *priv_dev = data; |
| irqreturn_t ret = IRQ_NONE; |
| unsigned long flags; |
| unsigned int bit; |
| unsigned long reg; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| |
| reg = readl(&priv_dev->regs->usb_ists); |
| if (reg) { |
| writel(reg, &priv_dev->regs->usb_ists); |
| writel(USB_IEN_INIT, &priv_dev->regs->usb_ien); |
| cdns3_check_usb_interrupt_proceed(priv_dev, reg); |
| ret = IRQ_HANDLED; |
| } |
| |
| reg = readl(&priv_dev->regs->ep_ists); |
| |
| /* handle default endpoint OUT */ |
| if (reg & EP_ISTS_EP_OUT0) { |
| cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT); |
| ret = IRQ_HANDLED; |
| } |
| |
| /* handle default endpoint IN */ |
| if (reg & EP_ISTS_EP_IN0) { |
| cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN); |
| ret = IRQ_HANDLED; |
| } |
| |
| /* check if interrupt from non default endpoint, if no exit */ |
| reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0); |
| if (!reg) |
| goto irqend; |
| |
| for_each_set_bit(bit, ®, |
| sizeof(u32) * BITS_PER_BYTE) { |
| cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]); |
| ret = IRQ_HANDLED; |
| } |
| |
| if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams) |
| cdns3_wa2_check_outq_status(priv_dev); |
| |
| irqend: |
| writel(~0, &priv_dev->regs->ep_ien); |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| |
| return ret; |
| } |
| |
| /** |
| * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP |
| * |
| * The real reservation will occur during write to EP_CFG register, |
| * this function is used to check if the 'size' reservation is allowed. |
| * |
| * @priv_dev: extended gadget object |
| * @size: the size (KB) for EP would like to allocate |
| * @is_in: endpoint direction |
| * |
| * Return 0 if the required size can met or negative value on failure |
| */ |
| static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev, |
| int size, int is_in) |
| { |
| int remained; |
| |
| /* 2KB are reserved for EP0*/ |
| remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2; |
| |
| if (is_in) { |
| if (remained < size) |
| return -EPERM; |
| |
| priv_dev->onchip_used_size += size; |
| } else { |
| int required; |
| |
| /** |
| * ALL OUT EPs are shared the same chunk onchip memory, so |
| * driver checks if it already has assigned enough buffers |
| */ |
| if (priv_dev->out_mem_is_allocated >= size) |
| return 0; |
| |
| required = size - priv_dev->out_mem_is_allocated; |
| |
| if (required > remained) |
| return -EPERM; |
| |
| priv_dev->out_mem_is_allocated += required; |
| priv_dev->onchip_used_size += required; |
| } |
| |
| return 0; |
| } |
| |
| static void cdns3_configure_dmult(struct cdns3_device *priv_dev, |
| struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_usb_regs __iomem *regs = priv_dev->regs; |
| |
| /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */ |
| if (priv_dev->dev_ver <= DEV_VER_V2) |
| writel(USB_CONF_DMULT, ®s->usb_conf); |
| |
| if (priv_dev->dev_ver == DEV_VER_V2) |
| writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2); |
| |
| if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) { |
| u32 mask; |
| |
| if (priv_ep->dir) |
| mask = BIT(priv_ep->num + 16); |
| else |
| mask = BIT(priv_ep->num); |
| |
| if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) { |
| cdns3_set_register_bit(®s->tdl_from_trb, mask); |
| cdns3_set_register_bit(®s->tdl_beh, mask); |
| cdns3_set_register_bit(®s->tdl_beh2, mask); |
| cdns3_set_register_bit(®s->dma_adv_td, mask); |
| } |
| |
| if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) |
| cdns3_set_register_bit(®s->tdl_from_trb, mask); |
| |
| cdns3_set_register_bit(®s->dtrans, mask); |
| } |
| } |
| |
| /** |
| * cdns3_ep_config - Configure hardware endpoint |
| * @priv_ep: extended endpoint object |
| * @enable: set EP_CFG_ENABLE bit in ep_cfg register. |
| */ |
| int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable) |
| { |
| bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC); |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| u32 bEndpointAddress = priv_ep->num | priv_ep->dir; |
| u32 max_packet_size = 0; |
| u8 maxburst = 0; |
| u32 ep_cfg = 0; |
| u8 buffering; |
| u8 mult = 0; |
| int ret; |
| |
| buffering = priv_dev->ep_buf_size - 1; |
| |
| cdns3_configure_dmult(priv_dev, priv_ep); |
| |
| switch (priv_ep->type) { |
| case USB_ENDPOINT_XFER_INT: |
| ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT); |
| |
| if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir) |
| ep_cfg |= EP_CFG_TDL_CHK; |
| break; |
| case USB_ENDPOINT_XFER_BULK: |
| ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK); |
| |
| if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir) |
| ep_cfg |= EP_CFG_TDL_CHK; |
| break; |
| default: |
| ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC); |
| mult = priv_dev->ep_iso_burst - 1; |
| buffering = mult + 1; |
| } |
| |
| switch (priv_dev->gadget.speed) { |
| case USB_SPEED_FULL: |
| max_packet_size = is_iso_ep ? 1023 : 64; |
| break; |
| case USB_SPEED_HIGH: |
| max_packet_size = is_iso_ep ? 1024 : 512; |
| break; |
| case USB_SPEED_SUPER: |
| /* It's limitation that driver assumes in driver. */ |
| mult = 0; |
| max_packet_size = 1024; |
| if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) { |
| maxburst = priv_dev->ep_iso_burst - 1; |
| buffering = (mult + 1) * |
| (maxburst + 1); |
| |
| if (priv_ep->interval > 1) |
| buffering++; |
| } else { |
| maxburst = priv_dev->ep_buf_size - 1; |
| } |
| break; |
| default: |
| /* all other speed are not supported */ |
| return -EINVAL; |
| } |
| |
| if (max_packet_size == 1024) |
| priv_ep->trb_burst_size = 128; |
| else if (max_packet_size >= 512) |
| priv_ep->trb_burst_size = 64; |
| else |
| priv_ep->trb_burst_size = 16; |
| |
| mult = min_t(u8, mult, EP_CFG_MULT_MAX); |
| buffering = min_t(u8, buffering, EP_CFG_BUFFERING_MAX); |
| maxburst = min_t(u8, maxburst, EP_CFG_MAXBURST_MAX); |
| |
| /* onchip buffer is only allocated before configuration */ |
| if (!priv_dev->hw_configured_flag) { |
| ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1, |
| !!priv_ep->dir); |
| if (ret) { |
| dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n"); |
| return ret; |
| } |
| } |
| |
| if (enable) |
| ep_cfg |= EP_CFG_ENABLE; |
| |
| if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) { |
| if (priv_dev->dev_ver >= DEV_VER_V3) { |
| u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0)); |
| |
| /* |
| * Stream capable endpoints are handled by using ep_tdl |
| * register. Other endpoints use TDL from TRB feature. |
| */ |
| cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb, |
| mask); |
| } |
| |
| /* Enable Stream Bit TDL chk and SID chk */ |
| ep_cfg |= EP_CFG_STREAM_EN | EP_CFG_TDL_CHK | EP_CFG_SID_CHK; |
| } |
| |
| ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) | |
| EP_CFG_MULT(mult) | |
| EP_CFG_BUFFERING(buffering) | |
| EP_CFG_MAXBURST(maxburst); |
| |
| cdns3_select_ep(priv_dev, bEndpointAddress); |
| writel(ep_cfg, &priv_dev->regs->ep_cfg); |
| priv_ep->flags |= EP_CONFIGURED; |
| |
| dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n", |
| priv_ep->name, ep_cfg); |
| |
| return 0; |
| } |
| |
| /* Find correct direction for HW endpoint according to description */ |
| static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc, |
| struct cdns3_endpoint *priv_ep) |
| { |
| return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) || |
| (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc)); |
| } |
| |
| static struct |
| cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev, |
| struct usb_endpoint_descriptor *desc) |
| { |
| struct usb_ep *ep; |
| struct cdns3_endpoint *priv_ep; |
| |
| list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { |
| unsigned long num; |
| int ret; |
| /* ep name pattern likes epXin or epXout */ |
| char c[2] = {ep->name[2], '\0'}; |
| |
| ret = kstrtoul(c, 10, &num); |
| if (ret) |
| return ERR_PTR(ret); |
| |
| priv_ep = ep_to_cdns3_ep(ep); |
| if (cdns3_ep_dir_is_correct(desc, priv_ep)) { |
| if (!(priv_ep->flags & EP_CLAIMED)) { |
| priv_ep->num = num; |
| return priv_ep; |
| } |
| } |
| } |
| |
| return ERR_PTR(-ENOENT); |
| } |
| |
| /* |
| * Cadence IP has one limitation that all endpoints must be configured |
| * (Type & MaxPacketSize) before setting configuration through hardware |
| * register, it means we can't change endpoints configuration after |
| * set_configuration. |
| * |
| * This function set EP_CLAIMED flag which is added when the gadget driver |
| * uses usb_ep_autoconfig to configure specific endpoint; |
| * When the udc driver receives set_configurion request, |
| * it goes through all claimed endpoints, and configure all endpoints |
| * accordingly. |
| * |
| * At usb_ep_ops.enable/disable, we only enable and disable endpoint through |
| * ep_cfg register which can be changed after set_configuration, and do |
| * some software operation accordingly. |
| */ |
| static struct |
| usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget, |
| struct usb_endpoint_descriptor *desc, |
| struct usb_ss_ep_comp_descriptor *comp_desc) |
| { |
| struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); |
| struct cdns3_endpoint *priv_ep; |
| unsigned long flags; |
| |
| priv_ep = cdns3_find_available_ep(priv_dev, desc); |
| if (IS_ERR(priv_ep)) { |
| dev_err(priv_dev->dev, "no available ep\n"); |
| return NULL; |
| } |
| |
| dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name); |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| priv_ep->endpoint.desc = desc; |
| priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT; |
| priv_ep->type = usb_endpoint_type(desc); |
| priv_ep->flags |= EP_CLAIMED; |
| priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; |
| |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| return &priv_ep->endpoint; |
| } |
| |
| /** |
| * cdns3_gadget_ep_alloc_request - Allocates request |
| * @ep: endpoint object associated with request |
| * @gfp_flags: gfp flags |
| * |
| * Returns allocated request address, NULL on allocation error |
| */ |
| struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep, |
| gfp_t gfp_flags) |
| { |
| struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); |
| struct cdns3_request *priv_req; |
| |
| priv_req = kzalloc(sizeof(*priv_req), gfp_flags); |
| if (!priv_req) |
| return NULL; |
| |
| priv_req->priv_ep = priv_ep; |
| |
| trace_cdns3_alloc_request(priv_req); |
| return &priv_req->request; |
| } |
| |
| /** |
| * cdns3_gadget_ep_free_request - Free memory occupied by request |
| * @ep: endpoint object associated with request |
| * @request: request to free memory |
| */ |
| void cdns3_gadget_ep_free_request(struct usb_ep *ep, |
| struct usb_request *request) |
| { |
| struct cdns3_request *priv_req = to_cdns3_request(request); |
| |
| if (priv_req->aligned_buf) |
| priv_req->aligned_buf->in_use = 0; |
| |
| trace_cdns3_free_request(priv_req); |
| kfree(priv_req); |
| } |
| |
| /** |
| * cdns3_gadget_ep_enable - Enable endpoint |
| * @ep: endpoint object |
| * @desc: endpoint descriptor |
| * |
| * Returns 0 on success, error code elsewhere |
| */ |
| static int cdns3_gadget_ep_enable(struct usb_ep *ep, |
| const struct usb_endpoint_descriptor *desc) |
| { |
| struct cdns3_endpoint *priv_ep; |
| struct cdns3_device *priv_dev; |
| const struct usb_ss_ep_comp_descriptor *comp_desc; |
| u32 reg = EP_STS_EN_TRBERREN; |
| u32 bEndpointAddress; |
| unsigned long flags; |
| int enable = 1; |
| int ret = 0; |
| int val; |
| |
| priv_ep = ep_to_cdns3_ep(ep); |
| priv_dev = priv_ep->cdns3_dev; |
| comp_desc = priv_ep->endpoint.comp_desc; |
| |
| if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { |
| dev_dbg(priv_dev->dev, "usbss: invalid parameters\n"); |
| return -EINVAL; |
| } |
| |
| if (!desc->wMaxPacketSize) { |
| dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n"); |
| return -EINVAL; |
| } |
| |
| if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED, |
| "%s is already enabled\n", priv_ep->name)) |
| return 0; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| |
| priv_ep->endpoint.desc = desc; |
| priv_ep->type = usb_endpoint_type(desc); |
| priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; |
| |
| if (priv_ep->interval > ISO_MAX_INTERVAL && |
| priv_ep->type == USB_ENDPOINT_XFER_ISOC) { |
| dev_err(priv_dev->dev, "Driver is limited to %d period\n", |
| ISO_MAX_INTERVAL); |
| |
| ret = -EINVAL; |
| goto exit; |
| } |
| |
| bEndpointAddress = priv_ep->num | priv_ep->dir; |
| cdns3_select_ep(priv_dev, bEndpointAddress); |
| |
| /* |
| * For some versions of controller at some point during ISO OUT traffic |
| * DMA reads Transfer Ring for the EP which has never got doorbell. |
| * This issue was detected only on simulation, but to avoid this issue |
| * driver add protection against it. To fix it driver enable ISO OUT |
| * endpoint before setting DRBL. This special treatment of ISO OUT |
| * endpoints are recommended by controller specification. |
| */ |
| if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) |
| enable = 0; |
| |
| if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) { |
| /* |
| * Enable stream support (SS mode) related interrupts |
| * in EP_STS_EN Register |
| */ |
| if (priv_dev->gadget.speed >= USB_SPEED_SUPER) { |
| reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN | |
| EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN | |
| EP_STS_EN_STREAMREN; |
| priv_ep->use_streams = true; |
| ret = cdns3_ep_config(priv_ep, enable); |
| priv_dev->using_streams |= true; |
| } |
| } else { |
| ret = cdns3_ep_config(priv_ep, enable); |
| } |
| |
| if (ret) |
| goto exit; |
| |
| ret = cdns3_allocate_trb_pool(priv_ep); |
| if (ret) |
| goto exit; |
| |
| bEndpointAddress = priv_ep->num | priv_ep->dir; |
| cdns3_select_ep(priv_dev, bEndpointAddress); |
| |
| trace_cdns3_gadget_ep_enable(priv_ep); |
| |
| writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); |
| |
| ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, |
| !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)), |
| 1, 1000); |
| |
| if (unlikely(ret)) { |
| cdns3_free_trb_pool(priv_ep); |
| ret = -EINVAL; |
| goto exit; |
| } |
| |
| /* enable interrupt for selected endpoint */ |
| cdns3_set_register_bit(&priv_dev->regs->ep_ien, |
| BIT(cdns3_ep_addr_to_index(bEndpointAddress))); |
| |
| if (priv_dev->dev_ver < DEV_VER_V2) |
| cdns3_wa2_enable_detection(priv_dev, priv_ep, reg); |
| |
| writel(reg, &priv_dev->regs->ep_sts_en); |
| |
| ep->desc = desc; |
| priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING | |
| EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN); |
| priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR; |
| priv_ep->wa1_set = 0; |
| priv_ep->enqueue = 0; |
| priv_ep->dequeue = 0; |
| reg = readl(&priv_dev->regs->ep_sts); |
| priv_ep->pcs = !!EP_STS_CCS(reg); |
| priv_ep->ccs = !!EP_STS_CCS(reg); |
| /* one TRB is reserved for link TRB used in DMULT mode*/ |
| priv_ep->free_trbs = priv_ep->num_trbs - 1; |
| exit: |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| |
| return ret; |
| } |
| |
| /** |
| * cdns3_gadget_ep_disable - Disable endpoint |
| * @ep: endpoint object |
| * |
| * Returns 0 on success, error code elsewhere |
| */ |
| static int cdns3_gadget_ep_disable(struct usb_ep *ep) |
| { |
| struct cdns3_endpoint *priv_ep; |
| struct cdns3_request *priv_req; |
| struct cdns3_device *priv_dev; |
| struct usb_request *request; |
| unsigned long flags; |
| int ret = 0; |
| u32 ep_cfg; |
| int val; |
| |
| if (!ep) { |
| pr_err("usbss: invalid parameters\n"); |
| return -EINVAL; |
| } |
| |
| priv_ep = ep_to_cdns3_ep(ep); |
| priv_dev = priv_ep->cdns3_dev; |
| |
| if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED), |
| "%s is already disabled\n", priv_ep->name)) |
| return 0; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| |
| trace_cdns3_gadget_ep_disable(priv_ep); |
| |
| cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); |
| |
| ep_cfg = readl(&priv_dev->regs->ep_cfg); |
| ep_cfg &= ~EP_CFG_ENABLE; |
| writel(ep_cfg, &priv_dev->regs->ep_cfg); |
| |
| /** |
| * Driver needs some time before resetting endpoint. |
| * It need waits for clearing DBUSY bit or for timeout expired. |
| * 10us is enough time for controller to stop transfer. |
| */ |
| readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val, |
| !(val & EP_STS_DBUSY), 1, 10); |
| writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); |
| |
| readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, |
| !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)), |
| 1, 1000); |
| if (unlikely(ret)) |
| dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n", |
| priv_ep->name); |
| |
| while (!list_empty(&priv_ep->pending_req_list)) { |
| request = cdns3_next_request(&priv_ep->pending_req_list); |
| |
| cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), |
| -ESHUTDOWN); |
| } |
| |
| while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { |
| priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); |
| |
| kfree(priv_req->request.buf); |
| cdns3_gadget_ep_free_request(&priv_ep->endpoint, |
| &priv_req->request); |
| list_del_init(&priv_req->list); |
| --priv_ep->wa2_counter; |
| } |
| |
| while (!list_empty(&priv_ep->deferred_req_list)) { |
| request = cdns3_next_request(&priv_ep->deferred_req_list); |
| |
| cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), |
| -ESHUTDOWN); |
| } |
| |
| priv_ep->descmis_req = NULL; |
| |
| ep->desc = NULL; |
| priv_ep->flags &= ~EP_ENABLED; |
| priv_ep->use_streams = false; |
| |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| |
| return ret; |
| } |
| |
| /** |
| * __cdns3_gadget_ep_queue - Transfer data on endpoint |
| * @ep: endpoint object |
| * @request: request object |
| * @gfp_flags: gfp flags |
| * |
| * Returns 0 on success, error code elsewhere |
| */ |
| static int __cdns3_gadget_ep_queue(struct usb_ep *ep, |
| struct usb_request *request, |
| gfp_t gfp_flags) |
| { |
| struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| struct cdns3_request *priv_req; |
| int ret = 0; |
| |
| request->actual = 0; |
| request->status = -EINPROGRESS; |
| priv_req = to_cdns3_request(request); |
| trace_cdns3_ep_queue(priv_req); |
| |
| if (priv_dev->dev_ver < DEV_VER_V2) { |
| ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep, |
| priv_req); |
| |
| if (ret == EINPROGRESS) |
| return 0; |
| } |
| |
| ret = cdns3_prepare_aligned_request_buf(priv_req); |
| if (ret < 0) |
| return ret; |
| |
| ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request, |
| usb_endpoint_dir_in(ep->desc)); |
| if (ret) |
| return ret; |
| |
| list_add_tail(&request->list, &priv_ep->deferred_req_list); |
| |
| /* |
| * For stream capable endpoint if prime irq flag is set then only start |
| * request. |
| * If hardware endpoint configuration has not been set yet then |
| * just queue request in deferred list. Transfer will be started in |
| * cdns3_set_hw_configuration. |
| */ |
| if (!request->stream_id) { |
| if (priv_dev->hw_configured_flag && |
| !(priv_ep->flags & EP_STALLED) && |
| !(priv_ep->flags & EP_STALL_PENDING)) |
| cdns3_start_all_request(priv_dev, priv_ep); |
| } else { |
| if (priv_dev->hw_configured_flag && priv_ep->prime_flag) |
| cdns3_start_all_request(priv_dev, priv_ep); |
| } |
| |
| return 0; |
| } |
| |
| static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, |
| gfp_t gfp_flags) |
| { |
| struct usb_request *zlp_request; |
| struct cdns3_endpoint *priv_ep; |
| struct cdns3_device *priv_dev; |
| unsigned long flags; |
| int ret; |
| |
| if (!request || !ep) |
| return -EINVAL; |
| |
| priv_ep = ep_to_cdns3_ep(ep); |
| priv_dev = priv_ep->cdns3_dev; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| |
| ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags); |
| |
| if (ret == 0 && request->zero && request->length && |
| (request->length % ep->maxpacket == 0)) { |
| struct cdns3_request *priv_req; |
| |
| zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC); |
| zlp_request->buf = priv_dev->zlp_buf; |
| zlp_request->length = 0; |
| |
| priv_req = to_cdns3_request(zlp_request); |
| priv_req->flags |= REQUEST_ZLP; |
| |
| dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n", |
| priv_ep->name); |
| ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags); |
| } |
| |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| return ret; |
| } |
| |
| /** |
| * cdns3_gadget_ep_dequeue - Remove request from transfer queue |
| * @ep: endpoint object associated with request |
| * @request: request object |
| * |
| * Returns 0 on success, error code elsewhere |
| */ |
| int cdns3_gadget_ep_dequeue(struct usb_ep *ep, |
| struct usb_request *request) |
| { |
| struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| struct usb_request *req, *req_temp; |
| struct cdns3_request *priv_req; |
| struct cdns3_trb *link_trb; |
| u8 req_on_hw_ring = 0; |
| unsigned long flags; |
| int ret = 0; |
| |
| if (!ep || !request || !ep->desc) |
| return -EINVAL; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| |
| priv_req = to_cdns3_request(request); |
| |
| trace_cdns3_ep_dequeue(priv_req); |
| |
| cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); |
| |
| list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list, |
| list) { |
| if (request == req) { |
| req_on_hw_ring = 1; |
| goto found; |
| } |
| } |
| |
| list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list, |
| list) { |
| if (request == req) |
| goto found; |
| } |
| |
| goto not_found; |
| |
| found: |
| link_trb = priv_req->trb; |
| |
| /* Update ring only if removed request is on pending_req_list list */ |
| if (req_on_hw_ring && link_trb) { |
| link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma + |
| ((priv_req->end_trb + 1) * TRB_SIZE))); |
| link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) | |
| TRB_TYPE(TRB_LINK) | TRB_CHAIN); |
| |
| if (priv_ep->wa1_trb == priv_req->trb) |
| cdns3_wa1_restore_cycle_bit(priv_ep); |
| } |
| |
| cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET); |
| |
| not_found: |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| return ret; |
| } |
| |
| /** |
| * __cdns3_gadget_ep_set_halt - Sets stall on selected endpoint |
| * Should be called after acquiring spin_lock and selecting ep |
| * @priv_ep: endpoint object to set stall on. |
| */ |
| void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| |
| trace_cdns3_halt(priv_ep, 1, 0); |
| |
| if (!(priv_ep->flags & EP_STALLED)) { |
| u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts); |
| |
| if (!(ep_sts_reg & EP_STS_DBUSY)) |
| cdns3_ep_stall_flush(priv_ep); |
| else |
| priv_ep->flags |= EP_STALL_PENDING; |
| } |
| } |
| |
| /** |
| * __cdns3_gadget_ep_clear_halt - Clears stall on selected endpoint |
| * Should be called after acquiring spin_lock and selecting ep |
| * @priv_ep: endpoint object to clear stall on |
| */ |
| int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep) |
| { |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| struct usb_request *request; |
| struct cdns3_request *priv_req; |
| struct cdns3_trb *trb = NULL; |
| struct cdns3_trb trb_tmp; |
| int ret; |
| int val; |
| |
| trace_cdns3_halt(priv_ep, 0, 0); |
| |
| request = cdns3_next_request(&priv_ep->pending_req_list); |
| if (request) { |
| priv_req = to_cdns3_request(request); |
| trb = priv_req->trb; |
| if (trb) { |
| trb_tmp = *trb; |
| trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE); |
| } |
| } |
| |
| writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd); |
| |
| /* wait for EPRST cleared */ |
| ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, |
| !(val & EP_CMD_EPRST), 1, 100); |
| if (ret) |
| return -EINVAL; |
| |
| priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING); |
| |
| if (request) { |
| if (trb) |
| *trb = trb_tmp; |
| |
| cdns3_rearm_transfer(priv_ep, 1); |
| } |
| |
| cdns3_start_all_request(priv_dev, priv_ep); |
| return ret; |
| } |
| |
| /** |
| * cdns3_gadget_ep_set_halt - Sets/clears stall on selected endpoint |
| * @ep: endpoint object to set/clear stall on |
| * @value: 1 for set stall, 0 for clear stall |
| * |
| * Returns 0 on success, error code elsewhere |
| */ |
| int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value) |
| { |
| struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); |
| struct cdns3_device *priv_dev = priv_ep->cdns3_dev; |
| unsigned long flags; |
| int ret = 0; |
| |
| if (!(priv_ep->flags & EP_ENABLED)) |
| return -EPERM; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| |
| cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); |
| |
| if (!value) { |
| priv_ep->flags &= ~EP_WEDGE; |
| ret = __cdns3_gadget_ep_clear_halt(priv_ep); |
| } else { |
| __cdns3_gadget_ep_set_halt(priv_ep); |
| } |
| |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| |
| return ret; |
| } |
| |
| extern const struct usb_ep_ops cdns3_gadget_ep0_ops; |
| |
| static const struct usb_ep_ops cdns3_gadget_ep_ops = { |
| .enable = cdns3_gadget_ep_enable, |
| .disable = cdns3_gadget_ep_disable, |
| .alloc_request = cdns3_gadget_ep_alloc_request, |
| .free_request = cdns3_gadget_ep_free_request, |
| .queue = cdns3_gadget_ep_queue, |
| .dequeue = cdns3_gadget_ep_dequeue, |
| .set_halt = cdns3_gadget_ep_set_halt, |
| .set_wedge = cdns3_gadget_ep_set_wedge, |
| }; |
| |
| /** |
| * cdns3_gadget_get_frame - Returns number of actual ITP frame |
| * @gadget: gadget object |
| * |
| * Returns number of actual ITP frame |
| */ |
| static int cdns3_gadget_get_frame(struct usb_gadget *gadget) |
| { |
| struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); |
| |
| return readl(&priv_dev->regs->usb_itpn); |
| } |
| |
| int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev) |
| { |
| enum usb_device_speed speed; |
| |
| speed = cdns3_get_speed(priv_dev); |
| |
| if (speed >= USB_SPEED_SUPER) |
| return 0; |
| |
| /* Start driving resume signaling to indicate remote wakeup. */ |
| writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf); |
| |
| return 0; |
| } |
| |
| static int cdns3_gadget_wakeup(struct usb_gadget *gadget) |
| { |
| struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); |
| unsigned long flags; |
| int ret = 0; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| ret = __cdns3_gadget_wakeup(priv_dev); |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| return ret; |
| } |
| |
| static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget, |
| int is_selfpowered) |
| { |
| struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); |
| unsigned long flags; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| priv_dev->is_selfpowered = !!is_selfpowered; |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| return 0; |
| } |
| |
| static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on) |
| { |
| struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); |
| |
| if (is_on) { |
| writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf); |
| } else { |
| writel(~0, &priv_dev->regs->ep_ists); |
| writel(~0, &priv_dev->regs->usb_ists); |
| writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); |
| } |
| |
| return 0; |
| } |
| |
| static void cdns3_gadget_config(struct cdns3_device *priv_dev) |
| { |
| struct cdns3_usb_regs __iomem *regs = priv_dev->regs; |
| u32 reg; |
| |
| cdns3_ep0_config(priv_dev); |
| |
| /* enable interrupts for endpoint 0 (in and out) */ |
| writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien); |
| |
| /* |
| * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1 |
| * revision of controller. |
| */ |
| if (priv_dev->dev_ver == DEV_VER_TI_V1) { |
| reg = readl(®s->dbg_link1); |
| |
| reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK; |
| reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) | |
| DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET; |
| writel(reg, ®s->dbg_link1); |
| } |
| |
| /* |
| * By default some platforms has set protected access to memory. |
| * This cause problem with cache, so driver restore non-secure |
| * access to memory. |
| */ |
| reg = readl(®s->dma_axi_ctrl); |
| reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) | |
| DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE); |
| writel(reg, ®s->dma_axi_ctrl); |
| |
| /* enable generic interrupt*/ |
| writel(USB_IEN_INIT, ®s->usb_ien); |
| writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf); |
| /* keep Fast Access bit */ |
| writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr); |
| |
| cdns3_configure_dmult(priv_dev, NULL); |
| } |
| |
| /** |
| * cdns3_gadget_udc_start - Gadget start |
| * @gadget: gadget object |
| * @driver: driver which operates on this gadget |
| * |
| * Returns 0 on success, error code elsewhere |
| */ |
| static int cdns3_gadget_udc_start(struct usb_gadget *gadget, |
| struct usb_gadget_driver *driver) |
| { |
| struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); |
| unsigned long flags; |
| enum usb_device_speed max_speed = driver->max_speed; |
| |
| spin_lock_irqsave(&priv_dev->lock, flags); |
| priv_dev->gadget_driver = driver; |
| |
| /* limit speed if necessary */ |
| max_speed = min(driver->max_speed, gadget->max_speed); |
| |
| switch (max_speed) { |
| case USB_SPEED_FULL: |
| writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf); |
| writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); |
| break; |
| case USB_SPEED_HIGH: |
| writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); |
| break; |
| case USB_SPEED_SUPER: |
| break; |
| default: |
| dev_err(priv_dev->dev, |
| "invalid maximum_speed parameter %d\n", |
| max_speed); |
| fallthrough; |
| case USB_SPEED_UNKNOWN: |
| /* default to superspeed */ |
| max_speed = USB_SPEED_SUPER; |
| break; |
| } |
| |
| cdns3_gadget_config(priv_dev); |
| spin_unlock_irqrestore(&priv_dev->lock, flags); |
| return 0; |
| } |
| |
| /** |
| * cdns3_gadget_udc_stop - Stops gadget |
| * @gadget: gadget object |
| * |
| * Returns 0 |
| */ |
| static int cdns3_gadget_udc_stop(struct usb_gadget *gadget) |
| { |
| struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); |
| struct cdns3_endpoint *priv_ep; |
| u32 bEndpointAddress; |
| struct usb_ep *ep; |
| int val; |
| |
| priv_dev->gadget_driver = NULL; |
| |
| priv_dev->onchip_used_size = 0; |
| priv_dev->out_mem_is_allocated = 0; |
| priv_dev->gadget.speed = USB_SPEED_UNKNOWN; |
| |
| list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { |
| priv_ep = ep_to_cdns3_ep(ep); |
| bEndpointAddress = priv_ep->num | priv_ep->dir; |
| cdns3_select_ep(priv_dev, bEndpointAddress); |
| writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); |
| readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, |
| !(val & EP_CMD_EPRST), 1, 100); |
| |
| priv_ep->flags &= ~EP_CLAIMED; |
| } |
| |
| /* disable interrupt for device */ |
| writel(0, &priv_dev->regs->usb_ien); |
| writel(0, &priv_dev->regs->usb_pwr); |
| writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); |
| |
| return 0; |
| } |
| |
| /** |
| * cdns3_gadget_check_config - ensure cdns3 can support the USB configuration |
| * @gadget: pointer to the USB gadget |
| * |
| * Used to record the maximum number of endpoints being used in a USB composite |
| * device. (across all configurations) This is to be used in the calculation |
| * of the TXFIFO sizes when resizing internal memory for individual endpoints. |
| * It will help ensured that the resizing logic reserves enough space for at |
| * least one max packet. |
| */ |
| static int cdns3_gadget_check_config(struct usb_gadget *gadget) |
| { |
| struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); |
| struct usb_ep *ep; |
| int n_in = 0; |
| int total; |
| |
| list_for_each_entry(ep, &gadget->ep_list, ep_list) { |
| if (ep->claimed && (ep->address & USB_DIR_IN)) |
| n_in++; |
| } |
| |
| /* 2KB are reserved for EP0, 1KB for out*/ |
| total = 2 + n_in + 1; |
| |
| if (total > priv_dev->onchip_buffers) |
| return -ENOMEM; |
| |
| priv_dev->ep_buf_size = priv_dev->ep_iso_burst = |
| (priv_dev->onchip_buffers - 2) / (n_in + 1); |
| |
| return 0; |
| } |
| |
| static const struct usb_gadget_ops cdns3_gadget_ops = { |
| .get_frame = cdns3_gadget_get_frame, |
| .wakeup = cdns3_gadget_wakeup, |
| .set_selfpowered = cdns3_gadget_set_selfpowered, |
| .pullup = cdns3_gadget_pullup, |
| .udc_start = cdns3_gadget_udc_start, |
| .udc_stop = cdns3_gadget_udc_stop, |
| .match_ep = cdns3_gadget_match_ep, |
| .check_config = cdns3_gadget_check_config, |
| }; |
| |
| static void cdns3_free_all_eps(struct cdns3_device *priv_dev) |
| { |
| int i; |
| |
| /* ep0 OUT point to ep0 IN. */ |
| priv_dev->eps[16] = NULL; |
| |
| for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) |
| if (priv_dev->eps[i]) { |
| cdns3_free_trb_pool(priv_dev->eps[i]); |
| devm_kfree(priv_dev->dev, priv_dev->eps[i]); |
| } |
| } |
| |
| /** |
| * cdns3_init_eps - Initializes software endpoints of gadget |
| * @priv_dev: extended gadget object |
| * |
| * Returns 0 on success, error code elsewhere |
| */ |
| static int cdns3_init_eps(struct cdns3_device *priv_dev) |
| { |
| u32 ep_enabled_reg, iso_ep_reg; |
| struct cdns3_endpoint *priv_ep; |
| int ep_dir, ep_number; |
| u32 ep_mask; |
| int ret = 0; |
| int i; |
| |
| /* Read it from USB_CAP3 to USB_CAP5 */ |
| ep_enabled_reg = readl(&priv_dev->regs->usb_cap3); |
| iso_ep_reg = readl(&priv_dev->regs->usb_cap4); |
| |
| dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n"); |
| |
| for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) { |
| ep_dir = i >> 4; /* i div 16 */ |
| ep_number = i & 0xF; /* i % 16 */ |
| ep_mask = BIT(i); |
| |
| if (!(ep_enabled_reg & ep_mask)) |
| continue; |
| |
| if (ep_dir && !ep_number) { |
| priv_dev->eps[i] = priv_dev->eps[0]; |
| continue; |
| } |
| |
| priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep), |
| GFP_KERNEL); |
| if (!priv_ep) |
| goto err; |
| |
| /* set parent of endpoint object */ |
| priv_ep->cdns3_dev = priv_dev; |
| priv_dev->eps[i] = priv_ep; |
| priv_ep->num = ep_number; |
| priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT; |
| |
| if (!ep_number) { |
| ret = cdns3_init_ep0(priv_dev, priv_ep); |
| if (ret) { |
| dev_err(priv_dev->dev, "Failed to init ep0\n"); |
| goto err; |
| } |
| } else { |
| snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s", |
| ep_number, !!ep_dir ? "in" : "out"); |
| priv_ep->endpoint.name = priv_ep->name; |
| |
| usb_ep_set_maxpacket_limit(&priv_ep->endpoint, |
| CDNS3_EP_MAX_PACKET_LIMIT); |
| priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS; |
| priv_ep->endpoint.ops = &cdns3_gadget_ep_ops; |
| if (ep_dir) |
| priv_ep->endpoint.caps.dir_in = 1; |
| else |
| priv_ep->endpoint.caps.dir_out = 1; |
| |
| if (iso_ep_reg & ep_mask) |
| priv_ep->endpoint.caps.type_iso = 1; |
| |
| priv_ep->endpoint.caps.type_bulk = 1; |
| priv_ep->endpoint.caps.type_int = 1; |
| |
| list_add_tail(&priv_ep->endpoint.ep_list, |
| &priv_dev->gadget.ep_list); |
| } |
| |
| priv_ep->flags = 0; |
| |
| dev_dbg(priv_dev->dev, "Initialized %s support: %s %s\n", |
| priv_ep->name, |
| priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "", |
| priv_ep->endpoint.caps.type_iso ? "ISO" : ""); |
| |
| INIT_LIST_HEAD(&priv_ep->pending_req_list); |
| INIT_LIST_HEAD(&priv_ep->deferred_req_list); |
| INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list); |
| } |
| |
| return 0; |
| err: |
| cdns3_free_all_eps(priv_dev); |
| return -ENOMEM; |
| } |
| |
| static void cdns3_gadget_release(struct device *dev) |
| { |
| struct cdns3_device *priv_dev = container_of(dev, |
| struct cdns3_device, gadget.dev); |
| |
| kfree(priv_dev); |
| } |
| |
| static void cdns3_gadget_exit(struct cdns *cdns) |
| { |
| struct cdns3_device *priv_dev; |
| |
| priv_dev = cdns->gadget_dev; |
| |
| |
| pm_runtime_mark_last_busy(cdns->dev); |
| pm_runtime_put_autosuspend(cdns->dev); |
| |
| usb_del_gadget(&priv_dev->gadget); |
| devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev); |
| |
| cdns3_free_all_eps(priv_dev); |
| |
| while (!list_empty(&priv_dev->aligned_buf_list)) { |
| struct cdns3_aligned_buf *buf; |
| |
| buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list); |
| dma_free_noncoherent(priv_dev->sysdev, buf->size, |
| buf->buf, |
| buf->dma, |
| buf->dir); |
| |
| list_del(&buf->list); |
| kfree(buf); |
| } |
| |
| dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf, |
| priv_dev->setup_dma); |
| dma_pool_destroy(priv_dev->eps_dma_pool); |
| |
| kfree(priv_dev->zlp_buf); |
| usb_put_gadget(&priv_dev->gadget); |
| cdns->gadget_dev = NULL; |
|